VictorSanh

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.optim import Optimizer

KD_loss = nn.KLDivLoss(reduction='batchmean')

def kd_step(teacher: nn.Module,
            student: nn.Module,
            temperature: float,

VictorSanh

# Cuda for Ubuntu18.04
CUDA_REPO_PKG=cuda-repo-ubuntu1804_10.1.243-1_amd64.deb
wget -O /tmp/${CUDA_REPO_PKG} http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/${CUDA_REPO_PKG} 
sudo dpkg -i /tmp/${CUDA_REPO_PKG}
sudo apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub 
rm -f /tmp/${CUDA_REPO_PKG}
sudo apt-get update
sudo apt-get install cuda-drivers -y
sudo apt-get install cuda -y

VictorSanh

import torchvision.models as models
resnet18 = models.resnet18()

from transformers import BertEmbeddings, BertEncoder


class MMBDEmbeddings(nn.Module):
    def __init__(self,
                 text_mod_embds = BertEmbeddings, # Or your favorite bidirectional transformer
                 vision_mod_embds = resnet18): # Or your favorite vision model

VictorSanh

import tensorflow as tf
from transformers import BertTokenizer, TFBertForSequenceClassification
import numpy as np


# seq_length = 128
# nb_examples = 1
# voc_size = 25000
# input_ids = tf.random.uniform((nb_examples,seq_length),
# 							maxval=voc_size,

VictorSanh

# Copyright (c) 2017-present, Moscow Institute of Physics and Technology.
# All rights reserved.
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.

from parlai.core.params import ParlaiParser
from parlai.core.agents import Agent
from parlai.core.utils import display_messages
from projects.convai2.models.ftlm.wild_eval_world import ConvAIWorld

VictorSanh

# coding: utf-8

import logging
import math
from typing import Any, Dict, List, Optional, Tuple

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn.parameter import Parameter, Variable