JohnGiorgi

#!/usr/bin/env python
# coding=utf-8
# Copyright 2021 The HuggingFace Team All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#

JohnGiorgi

import datetime
import logging
import math
import os
import re
import time
import traceback
from typing import Dict, List, Optional, Tuple, Union, Any

from apex import amp

JohnGiorgi

import logging
from typing import Dict

from allennlp.data.dataset_readers import SnliReader
from allennlp.data.dataset_readers.dataset_reader import DatasetReader
from allennlp.data.fields import Field
from allennlp.data.fields import LabelField
from allennlp.data.fields import TextField
from allennlp.data.instance import Instance
from allennlp.data.token_indexers import TokenIndexer

JohnGiorgi

import torch
from torch.nn.modules.dropout import _DropoutNd


class WordDropout(_DropoutNd):
    """During training, randomly replaces some of the elements of the input tensor with
    `dropout_constant` with probability `p` using samples from a Bernoulli distriution. Each channel
    will be zerored out independently on every forward call.

    Input is expected to be a 2D tensor of indices representing tokenized sentences.

JohnGiorgi

import torch


class BiaffineAttention(torch.nn.Module):
    """Implements a biaffine attention operator for binary relation classification.

    PyTorch implementation of the biaffine attention operator from "End-to-end neural relation
    extraction using deep biaffine attention" (https://arxiv.org/abs/1812.11275) which can be used
    as a classifier for binary relation classification.

JohnGiorgi

Lanugage Models for Fine-Tuning

BERT

Tensorflow

• Original Repository

Keras

JohnGiorgi

Setting up an Enviornment for Fastai

TOC

• Google Colab Setup
• Local Setup

Useful links

• The fastai forums

JohnGiorgi

Using Compute Canada Resources

Basics

You ssh into the systems using:

ssh <username>@<login node>

E.g., to login to Niagra for user example:

JohnGiorgi

Setting up a GPU-enabled Tensorflow Enviorment

Setting up GPU support

Training Tensorflow models on a NVIDIA GPU requires NVIDIA drivers, CUDA, and cuDNN. See here for the instructions to install all of these components on Ubuntu 18.04.

Other resources

• Installing NVIDIA drivers