notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import tqdm\n",
    "import math\n",
    "import sqlite3\n",
    "\n",
    "os.environ['CUDA_LAUNCH_BLOCKING'] = '1'\n",
    "os.environ['TOKENIZERS_PARALLELISM'] = 'true'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer_folder = \"tokenizer\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def sqlite_to_df(db_path, db_name=\"patent\"):\n",
    "#     conn = sqlite3.connect(db_path)\n",
    "#     df = pd.read_sql(f\"SELECT * FROM {db_name}\", conn)\n",
    "#     conn.close()\n",
    "#     return df\n",
    "\n",
    "# df = sqlite_to_df(\"/home/alfos/workspace/darpa/darpa-topic-modeling/models/patent_large.db\")\n",
    "\n",
    "# df = df.assign(text_id=df.apply(lambda row: row.patent_number + '_' + row.section, axis=1))\n",
    "\n",
    "# patent_text = df.groupby(\"text_id\").text.transform(lambda txt: \" \".join(txt))\n",
    "# df = df.assign(patent_text=patent_text)\n",
    "# df = df.drop_duplicates(\"text_id\").reset_index(drop=True)\n",
    "# df = df[['cpcs', 'title', 'patent_number', 'section', 'patent_text']]\n",
    "\n",
    "# train = df.sample(frac=0.8)\n",
    "# test = df[~df.index.isin(train.index)]\n",
    "\n",
    "# train.to_csv(\"data/train.csv\", index=False)\n",
    "# test.to_csv(\"data/test.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num Train Examples: 77174\n",
      "Num Test Examples: 19294\n"
     ]
    }
   ],
   "source": [
    "train = pd.read_csv(\"data/train.csv\")\n",
    "test = pd.read_csv(\"data/test.csv\")\n",
    "\n",
    "print(f\"Num Train Examples: {len(train)}\")\n",
    "print(f\"Num Test Examples: {len(test)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build a Tokenizer\n",
    "### Create the dataset to train a tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "txt_files_dir = \"./text_split\"\n",
    "!rm -rf {txt_files_dir}\n",
    "!mkdir {txt_files_dir}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def column_to_files(column, prefix, txt_files_dir):\n",
    "    # The prefix is a unique ID to avoid to overwrite a text file\n",
    "    i = prefix\n",
    "    # For every value in the df, with just one column\n",
    "    for row in column.to_list():\n",
    "        # Create the filename using the prefix ID\n",
    "        file_name = os.path.join(txt_files_dir, str(i) + '.txt')\n",
    "\n",
    "        try:\n",
    "            # Create the file and write the column text to it\n",
    "            f = open(file_name, \"wb\")\n",
    "            f.write(row.encode(\"utf-8\"))\n",
    "            f.close()\n",
    "        except Exception as e:\n",
    "            print(row, e)\n",
    "        i += 1\n",
    "    # Return the last prefix ID\n",
    "    return i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "77174\n"
     ]
    }
   ],
   "source": [
    "data = train['patent_text']\n",
    "\n",
    "# Set the ID to 0\n",
    "prefix = 0\n",
    "# Create a file for every patent text in train\n",
    "prefix = column_to_files(data, prefix, txt_files_dir)\n",
    "# Print the last ID\n",
    "print(prefix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "96468\n"
     ]
    }
   ],
   "source": [
    "data = test['patent_text']\n",
    "\n",
    "# Create a file for every patent text in test\n",
    "prefix = column_to_files(data, prefix, txt_files_dir)\n",
    "# Print the last ID\n",
    "print(prefix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cpcs</th>\n",
       "      <th>title</th>\n",
       "      <th>patent_number</th>\n",
       "      <th>section</th>\n",
       "      <th>patent_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>G06F 16/2365,G06F 16/128,G06F 16/24561</td>\n",
       "      <td>Methods, apparatuses, and systems for ingestin...</td>\n",
       "      <td>US11269856B2</td>\n",
       "      <td>background</td>\n",
       "      <td>The inventors have discovered limitations with...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C12N 9/0042</td>\n",
       "      <td>Recombinant vector and method for producing re...</td>\n",
       "      <td>US11236309B2</td>\n",
       "      <td>abstract</td>\n",
       "      <td>A recombinant vector according to an embodimen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C23C 16/345,B01J 29/40,C23C 16/0254,C23C 16/36...</td>\n",
       "      <td>Processes for depositing silicon-containing fi...</td>\n",
       "      <td>US11268190B2</td>\n",
       "      <td>summary</td>\n",
       "      <td>Described herein are halidosilane compounds, p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>F24F 7/10,E04B 1/7069,E04B 1/72,F24F 7/02,F24F...</td>\n",
       "      <td>Systems and methods for controlling air proper...</td>\n",
       "      <td>US11248814B2</td>\n",
       "      <td>abstract</td>\n",
       "      <td>A structure comprises at least one outer wall ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>G06F 16/2379,G06F 16/221,G06F 16/2455</td>\n",
       "      <td>Scalable implementations of exact distinct cou...</td>\n",
       "      <td>US11263202B2</td>\n",
       "      <td>abstract</td>\n",
       "      <td>Scalable implementations of exact distinct cou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77169</th>\n",
       "      <td>B65D 19/0012,B65D 2519/00019,B65D 2519/00034,B...</td>\n",
       "      <td>Stackable pallet display</td>\n",
       "      <td>US11267609B2</td>\n",
       "      <td>background</td>\n",
       "      <td>The packaging industry is always striving to i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77170</th>\n",
       "      <td>A01J 25/02,A01J 25/115,A01J 25/15,A47J 27/004,...</td>\n",
       "      <td>Countertop cooking appliance</td>\n",
       "      <td>US11259497B2</td>\n",
       "      <td>abstract</td>\n",
       "      <td>A countertop cooking appliance is an apparatus...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77171</th>\n",
       "      <td>H04B 7/0626,H04B 7/0617,H04B 7/0857,H04B 17/336</td>\n",
       "      <td>Coordinated beamforming of transmitting nodes ...</td>\n",
       "      <td>US11251850B1</td>\n",
       "      <td>background</td>\n",
       "      <td>Wireless mesh networks include a collection of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77172</th>\n",
       "      <td>G09G 5/14,G09G 2320/0247,G09G 2320/0673</td>\n",
       "      <td>Electronic device and method for driving displ...</td>\n",
       "      <td>US11238832B2</td>\n",
       "      <td>summary</td>\n",
       "      <td>In displaying a screen on a display in a recen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77173</th>\n",
       "      <td>G06Q 20/3227,G06Q 20/206,G06Q 30/0603,G06Q 30/...</td>\n",
       "      <td>Mobile cart reconciliation</td>\n",
       "      <td>US11244301B2</td>\n",
       "      <td>abstract</td>\n",
       "      <td>Disclosed herein is a method where a consumer ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>77174 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    cpcs  \\\n",
       "0                 G06F 16/2365,G06F 16/128,G06F 16/24561   \n",
       "1                                            C12N 9/0042   \n",
       "2      C23C 16/345,B01J 29/40,C23C 16/0254,C23C 16/36...   \n",
       "3      F24F 7/10,E04B 1/7069,E04B 1/72,F24F 7/02,F24F...   \n",
       "4                  G06F 16/2379,G06F 16/221,G06F 16/2455   \n",
       "...                                                  ...   \n",
       "77169  B65D 19/0012,B65D 2519/00019,B65D 2519/00034,B...   \n",
       "77170  A01J 25/02,A01J 25/115,A01J 25/15,A47J 27/004,...   \n",
       "77171    H04B 7/0626,H04B 7/0617,H04B 7/0857,H04B 17/336   \n",
       "77172            G09G 5/14,G09G 2320/0247,G09G 2320/0673   \n",
       "77173  G06Q 20/3227,G06Q 20/206,G06Q 30/0603,G06Q 30/...   \n",
       "\n",
       "                                                   title patent_number  \\\n",
       "0      Methods, apparatuses, and systems for ingestin...  US11269856B2   \n",
       "1      Recombinant vector and method for producing re...  US11236309B2   \n",
       "2      Processes for depositing silicon-containing fi...  US11268190B2   \n",
       "3      Systems and methods for controlling air proper...  US11248814B2   \n",
       "4      Scalable implementations of exact distinct cou...  US11263202B2   \n",
       "...                                                  ...           ...   \n",
       "77169                           Stackable pallet display  US11267609B2   \n",
       "77170                       Countertop cooking appliance  US11259497B2   \n",
       "77171  Coordinated beamforming of transmitting nodes ...  US11251850B1   \n",
       "77172  Electronic device and method for driving displ...  US11238832B2   \n",
       "77173                         Mobile cart reconciliation  US11244301B2   \n",
       "\n",
       "          section                                        patent_text  \n",
       "0      background  The inventors have discovered limitations with...  \n",
       "1        abstract  A recombinant vector according to an embodimen...  \n",
       "2         summary  Described herein are halidosilane compounds, p...  \n",
       "3        abstract  A structure comprises at least one outer wall ...  \n",
       "4        abstract  Scalable implementations of exact distinct cou...  \n",
       "...           ...                                                ...  \n",
       "77169  background  The packaging industry is always striving to i...  \n",
       "77170    abstract  A countertop cooking appliance is an apparatus...  \n",
       "77171  background  Wireless mesh networks include a collection of...  \n",
       "77172     summary  In displaying a screen on a display in a recen...  \n",
       "77173    abstract  Disclosed herein is a method where a consumer ...  \n",
       "\n",
       "[77174 rows x 5 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train the tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: transformers in ./env/lib/python3.9/site-packages (4.18.0)\n",
      "Requirement already satisfied: regex!=2019.12.17 in ./env/lib/python3.9/site-packages (from transformers) (2022.4.24)\n",
      "Requirement already satisfied: tqdm>=4.27 in ./env/lib/python3.9/site-packages (from transformers) (4.64.0)\n",
      "Requirement already satisfied: sacremoses in ./env/lib/python3.9/site-packages (from transformers) (0.0.53)\n",
      "Requirement already satisfied: requests in ./env/lib/python3.9/site-packages (from transformers) (2.27.1)\n",
      "Requirement already satisfied: pyyaml>=5.1 in ./env/lib/python3.9/site-packages (from transformers) (6.0)\n",
      "Requirement already satisfied: numpy>=1.17 in ./env/lib/python3.9/site-packages (from transformers) (1.22.3)\n",
      "Requirement already satisfied: tokenizers!=0.11.3,<0.13,>=0.11.1 in ./env/lib/python3.9/site-packages (from transformers) (0.12.1)\n",
      "Requirement already satisfied: filelock in ./env/lib/python3.9/site-packages (from transformers) (3.6.0)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in ./env/lib/python3.9/site-packages (from transformers) (0.5.1)\n",
      "Requirement already satisfied: packaging>=20.0 in ./env/lib/python3.9/site-packages (from transformers) (21.3)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in ./env/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (4.2.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in ./env/lib/python3.9/site-packages (from packaging>=20.0->transformers) (3.0.9)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in ./env/lib/python3.9/site-packages (from requests->transformers) (1.26.9)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in ./env/lib/python3.9/site-packages (from requests->transformers) (2021.10.8)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in ./env/lib/python3.9/site-packages (from requests->transformers) (2.0.12)\n",
      "Requirement already satisfied: idna<4,>=2.5 in ./env/lib/python3.9/site-packages (from requests->transformers) (3.3)\n",
      "Requirement already satisfied: six in ./env/lib/python3.9/site-packages (from sacremoses->transformers) (1.16.0)\n",
      "Requirement already satisfied: click in ./env/lib/python3.9/site-packages (from sacremoses->transformers) (8.1.3)\n",
      "Requirement already satisfied: joblib in ./env/lib/python3.9/site-packages (from sacremoses->transformers) (1.1.0)\n",
      "Requirement already satisfied: datasets in ./env/lib/python3.9/site-packages (2.2.0)\n",
      "Requirement already satisfied: dill in ./env/lib/python3.9/site-packages (from datasets) (0.3.4)\n",
      "Requirement already satisfied: packaging in ./env/lib/python3.9/site-packages (from datasets) (21.3)\n",
      "Requirement already satisfied: xxhash in ./env/lib/python3.9/site-packages (from datasets) (3.0.0)\n",
      "Requirement already satisfied: responses<0.19 in ./env/lib/python3.9/site-packages (from datasets) (0.18.0)\n",
      "Requirement already satisfied: numpy>=1.17 in ./env/lib/python3.9/site-packages (from datasets) (1.22.3)\n",
      "Requirement already satisfied: pyarrow>=6.0.0 in ./env/lib/python3.9/site-packages (from datasets) (8.0.0)\n",
      "Requirement already satisfied: fsspec[http]>=2021.05.0 in ./env/lib/python3.9/site-packages (from datasets) (2022.3.0)\n",
      "Requirement already satisfied: huggingface-hub<1.0.0,>=0.1.0 in ./env/lib/python3.9/site-packages (from datasets) (0.5.1)\n",
      "Requirement already satisfied: requests>=2.19.0 in ./env/lib/python3.9/site-packages (from datasets) (2.27.1)\n",
      "Requirement already satisfied: pandas in ./env/lib/python3.9/site-packages (from datasets) (1.4.2)\n",
      "Requirement already satisfied: aiohttp in ./env/lib/python3.9/site-packages (from datasets) (3.8.1)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in ./env/lib/python3.9/site-packages (from datasets) (4.64.0)\n",
      "Requirement already satisfied: multiprocess in ./env/lib/python3.9/site-packages (from datasets) (0.70.12.2)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in ./env/lib/python3.9/site-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (4.2.0)\n",
      "Requirement already satisfied: pyyaml in ./env/lib/python3.9/site-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (6.0)\n",
      "Requirement already satisfied: filelock in ./env/lib/python3.9/site-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.6.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in ./env/lib/python3.9/site-packages (from packaging->datasets) (3.0.9)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in ./env/lib/python3.9/site-packages (from requests>=2.19.0->datasets) (2021.10.8)\n",
      "Requirement already satisfied: idna<4,>=2.5 in ./env/lib/python3.9/site-packages (from requests>=2.19.0->datasets) (3.3)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in ./env/lib/python3.9/site-packages (from requests>=2.19.0->datasets) (1.26.9)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in ./env/lib/python3.9/site-packages (from requests>=2.19.0->datasets) (2.0.12)\n",
      "Requirement already satisfied: attrs>=17.3.0 in ./env/lib/python3.9/site-packages (from aiohttp->datasets) (21.4.0)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in ./env/lib/python3.9/site-packages (from aiohttp->datasets) (4.0.2)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in ./env/lib/python3.9/site-packages (from aiohttp->datasets) (1.3.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in ./env/lib/python3.9/site-packages (from aiohttp->datasets) (6.0.2)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in ./env/lib/python3.9/site-packages (from aiohttp->datasets) (1.7.2)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in ./env/lib/python3.9/site-packages (from aiohttp->datasets) (1.2.0)\n",
      "Requirement already satisfied: python-dateutil>=2.8.1 in ./env/lib/python3.9/site-packages (from pandas->datasets) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in ./env/lib/python3.9/site-packages (from pandas->datasets) (2022.1)\n",
      "Requirement already satisfied: six>=1.5 in ./env/lib/python3.9/site-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n",
      "Requirement already satisfied: torch in ./env/lib/python3.9/site-packages (1.11.0)\n",
      "Requirement already satisfied: typing-extensions in ./env/lib/python3.9/site-packages (from torch) (4.2.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers\n",
    "!pip install datasets\n",
    "!pip install torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from pathlib import Path\n",
    "\n",
    "# from tokenizers import ByteLevelBPETokenizer\n",
    "# from tokenizers.processors import BertProcessing\n",
    "\n",
    "import torch\n",
    "from torch.utils.data.dataset import Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%time\n",
    "# paths = [str(x) for x in Path(\".\").glob(\"text_split/*.txt\")]\n",
    "\n",
    "# # Initializer a tokenizer\n",
    "# tokenizer = ByteLevelBPETokenizer(lowercase=True)\n",
    "\n",
    "# # Customizer training\n",
    "# tokenizer.train(\n",
    "#     files=paths,\n",
    "#     vocab_size=8192,\n",
    "#     min_frequency=2,\n",
    "#     show_progress=True,\n",
    "#     special_tokens=[\"<s>\", \"<pad>\", \"</s>\", \"<mask>\"],)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Save the Tokenizer to dist\n",
    "# !rm -rf {tokenizer_folder}\n",
    "# !mkdir {tokenizer_folder}\n",
    "\n",
    "# tokenizer.save_model(tokenizer_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Prepare the tokenizer\n",
    "# tokenizer = ByteLevelBPETokenizer(\n",
    "#     os.path.abspath(os.path.join(tokenizer_folder, \"vocab.json\")),\n",
    "#     os.path.abspath(os.path.join(tokenizer_folder, \"merges.txt\"))\n",
    "# )\n",
    "# tokenizer.enable_truncation(max_length=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tokenizer.encode(train.patent_text.iloc[0]).tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train a language model from scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_BATCH_SIZE = 16\n",
    "VALID_BATCH_SIZE = 8\n",
    "TRAIN_EPOCHS = 15\n",
    "LEARNING_RATE = 1e-4\n",
    "WEIGHT_DECAY = 0.01\n",
    "SEED = 42\n",
    "MAX_LEN = 128\n",
    "SUMMARY_LEN = 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Thu May 12 16:56:25 2022       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 465.19.01    Driver Version: 465.19.01    CUDA Version: 11.3     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                               |                      |               MIG M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  NVIDIA GeForce ...  Off  | 00000000:01:00.0 Off |                  N/A |\n",
      "|  0%   39C    P8     7W / 198W |     42MiB /  8119MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                                  |\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
      "|        ID   ID                                                   Usage      |\n",
      "|=============================================================================|\n",
      "|    0   N/A  N/A      1442      G   /usr/lib/xorg/Xorg                 39MiB |\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "# Check we have a GPU\n",
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check that PyTorch sees it\n",
    "import torch\n",
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import RobertaConfig\n",
    "\n",
    "config = RobertaConfig(\n",
    "    vocab_size=8192,\n",
    "    max_position_embeddings=514,\n",
    "    num_attention_heads=12,\n",
    "    num_hidden_layers=6,\n",
    "    type_vocab_size=1,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num parameters: 49816064\n"
     ]
    }
   ],
   "source": [
    "from transformers import RobertaForMaskedLM\n",
    "\n",
    "model = RobertaForMaskedLM(config=config)\n",
    "print(f\"Num parameters: {model.num_parameters()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "from transformers import RobertaTokenizerFast\n",
    "\n",
    "# Create the tokenizer from a trained one\n",
    "tokenizer = RobertaTokenizerFast.from_pretrained(tokenizer_folder, max_len=MAX_LEN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PreTrainedTokenizerFast(name_or_path='tokenizer', vocab_size=8192, model_max_len=128, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'bos_token': AddedToken(\"<s>\", rstrip=False, lstrip=False, single_word=False, normalized=True), 'eos_token': AddedToken(\"</s>\", rstrip=False, lstrip=False, single_word=False, normalized=True), 'unk_token': AddedToken(\"<unk>\", rstrip=False, lstrip=False, single_word=False, normalized=True), 'sep_token': AddedToken(\"</s>\", rstrip=False, lstrip=False, single_word=False, normalized=True), 'pad_token': AddedToken(\"<pad>\", rstrip=False, lstrip=False, single_word=False, normalized=True), 'cls_token': AddedToken(\"<s>\", rstrip=False, lstrip=False, single_word=False, normalized=True), 'mask_token': AddedToken(\"<mask>\", rstrip=False, lstrip=True, single_word=False, normalized=True)})"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Building the training Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomDataset(Dataset):\n",
    "    def __init__(self, df, tokenizer):\n",
    "        self.examples = []\n",
    "        for example in df.values:\n",
    "            x = tokenizer.encode_plus(example, max_length=MAX_LEN, truncation=True, padding=True)\n",
    "            self.examples += [x.input_ids]\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.examples)\n",
    "\n",
    "    def __getitem__(self, i):\n",
    "        return torch.tensor(self.examples[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = CustomDataset(train.patent_text, tokenizer)\n",
    "eval_dataset = CustomDataset(test.patent_text, tokenizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define the Data Collator for masking our language"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import DataCollatorForLanguageModeling\n",
    "\n",
    "# Define the Data Collator\n",
    "data_collator = DataCollatorForLanguageModeling(\n",
    "    tokenizer=tokenizer, mlm=True, mlm_probability=0.15,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initializer and train our Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_folder = \"RoBERT\"\n",
    "\n",
    "!rm -rf {model_folder}\n",
    "!mkdir {model_folder}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RoBERT\n"
     ]
    }
   ],
   "source": [
    "from transformers import Trainer, TrainingArguments\n",
    "\n",
    "print(model_folder)\n",
    "# Define the training arguments\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=model_folder,\n",
    "    overwrite_output_dir=True,\n",
    "    evaluation_strategy=\"epoch\",\n",
    "    num_train_epochs=TRAIN_EPOCHS,\n",
    "    learning_rate=LEARNING_RATE,\n",
    "    weight_decay=WEIGHT_DECAY,\n",
    "    per_device_train_batch_size=TRAIN_BATCH_SIZE,\n",
    "    per_device_eval_batch_size=VALID_BATCH_SIZE,\n",
    "    save_steps=8192,\n",
    "    #eval_steps=4096,\n",
    "    save_total_limit=2,\n",
    ")\n",
    "# Create the Trainer for out model\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    data_collator=data_collator,\n",
    "    train_dataset=train_dataset,\n",
    "    eval_dataset=eval_dataset,\n",
    "    #prediction_loss_only=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "***** Running training *****\n",
      "  Num examples = 77174\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 16\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 72360\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aec8c68996894fa99e858009e1407589",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/72360 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 6.5111, 'learning_rate': 9.930901050304035e-05, 'epoch': 0.1}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [0,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [1,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [2,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [3,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [4,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [5,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [6,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [7,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [8,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [9,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [10,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [11,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [12,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [13,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [14,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [15,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [16,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [17,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [18,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [19,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [20,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [21,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [22,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [23,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [24,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [25,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [26,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [27,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [28,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [29,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [30,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [31,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [96,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [97,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [98,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [99,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [100,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [101,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [102,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [103,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [104,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [105,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [106,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [107,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [108,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [109,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [110,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [111,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [112,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [113,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [114,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [115,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [116,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [117,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [118,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [119,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [120,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [121,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [122,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [123,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [124,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [125,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [126,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [127,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [32,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [33,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [34,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [35,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [36,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [37,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [38,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [39,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [40,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [41,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [42,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [43,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [44,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [45,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [46,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [47,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [48,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [49,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [50,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [51,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [52,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [53,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [54,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [55,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [56,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [57,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [58,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [59,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [60,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [61,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [62,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [63,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [64,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [65,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [66,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [67,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [68,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [69,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [70,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [71,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [72,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [73,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [74,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [75,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [76,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [77,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [78,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [79,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [80,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [81,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [82,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [83,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [84,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [85,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [86,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [87,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [88,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [89,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [90,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [91,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [92,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [93,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [94,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [95,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [64,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [65,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [66,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [67,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [68,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [69,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [70,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [71,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [72,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [73,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [74,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [75,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [76,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [77,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [78,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [79,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [80,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [81,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [82,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [83,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [84,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [85,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [86,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [87,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [88,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [89,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [90,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [91,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [92,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [93,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [94,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [23,0,0], thread: [95,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [32,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [33,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [34,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [35,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [36,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [37,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [38,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [39,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [40,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [41,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [42,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [43,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [44,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [45,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [46,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [47,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [48,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [49,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [50,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [51,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [52,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [53,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [54,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [55,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [56,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [57,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [58,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [59,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [60,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [61,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [62,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [63,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [32,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [33,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [34,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [35,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [36,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [37,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [38,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [39,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [40,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [41,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [42,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [43,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [44,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [45,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [46,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [47,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [48,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [49,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [50,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [51,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [52,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [53,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [54,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [55,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [56,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [57,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [58,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [59,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [60,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [61,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [62,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [63,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [0,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [1,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [2,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [3,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [4,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [5,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [6,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [7,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [8,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [9,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [10,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [11,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [12,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [13,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [14,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [15,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [16,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [17,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [18,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [19,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [20,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [21,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [22,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [23,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [24,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [25,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [26,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [27,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [28,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [29,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [30,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [24,0,0], thread: [31,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [32,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [33,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [34,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [35,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [36,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [37,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [38,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [39,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [40,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [41,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [42,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [43,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [44,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [45,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [46,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [47,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [48,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [49,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [50,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [51,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [52,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [53,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [54,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [55,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [56,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [57,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [58,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [59,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [60,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [61,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [62,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [63,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [96,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [97,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [98,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [99,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [100,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [101,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [102,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [103,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [104,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [105,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [106,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [107,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [108,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [109,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [110,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [111,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [112,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [113,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [114,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [115,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [116,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [117,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [118,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [119,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [120,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [121,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [122,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [123,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [124,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [125,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [126,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [127,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [96,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [97,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [98,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [99,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [100,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [101,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [102,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [103,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [104,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [105,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [106,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [107,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [108,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [109,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [110,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [111,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [112,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [113,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [114,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [115,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [116,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [117,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [118,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [119,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [120,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [121,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [122,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [123,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [124,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [125,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [126,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [20,0,0], thread: [127,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [96,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [97,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [98,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [99,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [100,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [101,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [102,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [103,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [104,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [105,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [106,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [107,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [108,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [109,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [110,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [111,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [112,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [113,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [114,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [115,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [116,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [117,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [118,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [119,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [120,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [121,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [122,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [123,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [124,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [125,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [126,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [127,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [64,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [65,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [66,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [67,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [68,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [69,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [70,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [71,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [72,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [73,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [74,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [75,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [76,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [77,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [78,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [79,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [80,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [81,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [82,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [83,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [84,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [85,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [86,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [87,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [88,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [89,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [90,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [91,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [92,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [93,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [94,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [22,0,0], thread: [95,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [64,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [65,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [66,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [67,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [68,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [69,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [70,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [71,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [72,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [73,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [74,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [75,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [76,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [77,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [78,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [79,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [80,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [81,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [82,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [83,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [84,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [85,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [86,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [87,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [88,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [89,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [90,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [91,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [92,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [93,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [94,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [95,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [0,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [1,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [2,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [3,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [4,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [5,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [6,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [7,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [8,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [9,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [10,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [11,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [12,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [13,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [14,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [15,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [16,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [17,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [18,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [19,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [20,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [21,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [22,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [23,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [24,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [25,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [26,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [27,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [28,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [29,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [30,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [31,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [0,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [1,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [2,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [3,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [4,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [5,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [6,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [7,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [8,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [9,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [10,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [11,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [12,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [13,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [14,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [15,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [16,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [17,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [18,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [19,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [20,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [21,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [22,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [23,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [24,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [25,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [26,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [27,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [28,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [29,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [30,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [21,0,0], thread: [31,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [64,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [65,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [66,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [67,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [68,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [69,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [70,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [71,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [72,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [73,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [74,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [75,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [76,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [77,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [78,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [79,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [80,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [81,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [82,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [83,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [84,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [85,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [86,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [87,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [88,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [89,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [90,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [91,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [92,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [93,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [94,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [95,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [32,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [33,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [34,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [35,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [36,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [37,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [38,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [39,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [40,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [41,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [42,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [43,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [44,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [45,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [46,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [47,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [48,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [49,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [50,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [51,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [52,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [53,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [54,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [55,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [56,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [57,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [58,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [59,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [60,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [61,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [62,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n",
      "../aten/src/ATen/native/cuda/Indexing.cu:703: indexSelectLargeIndex: block: [25,0,0], thread: [63,0,0] Assertion `srcIndex < srcSelectDimSize` failed.\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "CUDA error: device-side assert triggered",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[1;32m/home/alfos/workspace/darpa/train_huggingface_from_scratch/notebook.ipynb Cell 37'\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      <a href='vscode-notebook-cell:/home/alfos/workspace/darpa/train_huggingface_from_scratch/notebook.ipynb#ch0000036?line=0'>1</a>\u001b[0m \u001b[39m# Train the model\u001b[39;00m\n\u001b[0;32m----> <a href='vscode-notebook-cell:/home/alfos/workspace/darpa/train_huggingface_from_scratch/notebook.ipynb#ch0000036?line=1'>2</a>\u001b[0m trainer\u001b[39m.\u001b[39;49mtrain()\n",
      "File \u001b[0;32m~/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/trainer.py:1422\u001b[0m, in \u001b[0;36mTrainer.train\u001b[0;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[0m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/trainer.py?line=1419'>1420</a>\u001b[0m         tr_loss_step \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtraining_step(model, inputs)\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/trainer.py?line=1420'>1421</a>\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/trainer.py?line=1421'>1422</a>\u001b[0m     tr_loss_step \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mtraining_step(model, inputs)\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/trainer.py?line=1423'>1424</a>\u001b[0m \u001b[39mif\u001b[39;00m (\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/trainer.py?line=1424'>1425</a>\u001b[0m     args\u001b[39m.\u001b[39mlogging_nan_inf_filter\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/trainer.py?line=1425'>1426</a>\u001b[0m     \u001b[39mand\u001b[39;00m \u001b[39mnot\u001b[39;00m is_torch_tpu_available()\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/trainer.py?line=1426'>1427</a>\u001b[0m     \u001b[39mand\u001b[39;00m (torch\u001b[39m.\u001b[39misnan(tr_loss_step) \u001b[39mor\u001b[39;00m torch\u001b[39m.\u001b[39misinf(tr_loss_step))\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/trainer.py?line=1427'>1428</a>\u001b[0m ):\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/trainer.py?line=1428'>1429</a>\u001b[0m     \u001b[39m# if loss is nan or inf simply add the average of previous logged losses\u001b[39;00m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/trainer.py?line=1429'>1430</a>\u001b[0m     tr_loss \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m tr_loss \u001b[39m/\u001b[39m (\u001b[39m1\u001b[39m \u001b[39m+\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstate\u001b[39m.\u001b[39mglobal_step \u001b[39m-\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_globalstep_last_logged)\n",
      "File \u001b[0;32m~/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/trainer.py:2011\u001b[0m, in \u001b[0;36mTrainer.training_step\u001b[0;34m(self, model, inputs)\u001b[0m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/trainer.py?line=2007'>2008</a>\u001b[0m     \u001b[39mreturn\u001b[39;00m loss_mb\u001b[39m.\u001b[39mreduce_mean()\u001b[39m.\u001b[39mdetach()\u001b[39m.\u001b[39mto(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39margs\u001b[39m.\u001b[39mdevice)\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/trainer.py?line=2009'>2010</a>\u001b[0m \u001b[39mwith\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mautocast_smart_context_manager():\n\u001b[0;32m-> <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/trainer.py?line=2010'>2011</a>\u001b[0m     loss \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mcompute_loss(model, inputs)\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/trainer.py?line=2012'>2013</a>\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39margs\u001b[39m.\u001b[39mn_gpu \u001b[39m>\u001b[39m \u001b[39m1\u001b[39m:\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/trainer.py?line=2013'>2014</a>\u001b[0m     loss \u001b[39m=\u001b[39m loss\u001b[39m.\u001b[39mmean()  \u001b[39m# mean() to average on multi-gpu parallel training\u001b[39;00m\n",
      "File \u001b[0;32m~/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/trainer.py:2043\u001b[0m, in \u001b[0;36mTrainer.compute_loss\u001b[0;34m(self, model, inputs, return_outputs)\u001b[0m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/trainer.py?line=2040'>2041</a>\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/trainer.py?line=2041'>2042</a>\u001b[0m     labels \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[0;32m-> <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/trainer.py?line=2042'>2043</a>\u001b[0m outputs \u001b[39m=\u001b[39m model(\u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49minputs)\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/trainer.py?line=2043'>2044</a>\u001b[0m \u001b[39m# Save past state if it exists\u001b[39;00m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/trainer.py?line=2044'>2045</a>\u001b[0m \u001b[39m# TODO: this needs to be fixed and made cleaner later.\u001b[39;00m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/trainer.py?line=2045'>2046</a>\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39margs\u001b[39m.\u001b[39mpast_index \u001b[39m>\u001b[39m\u001b[39m=\u001b[39m \u001b[39m0\u001b[39m:\n",
      "File \u001b[0;32m~/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py:1110\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1105'>1106</a>\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1106'>1107</a>\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1107'>1108</a>\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1108'>1109</a>\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1109'>1110</a>\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1110'>1111</a>\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1111'>1112</a>\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n",
      "File \u001b[0;32m~/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py:1098\u001b[0m, in \u001b[0;36mRobertaForMaskedLM.forward\u001b[0;34m(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, labels, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=1087'>1088</a>\u001b[0m \u001b[39mr\u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=1088'>1089</a>\u001b[0m \u001b[39mlabels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):\u001b[39;00m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=1089'>1090</a>\u001b[0m \u001b[39m    Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=1093'>1094</a>\u001b[0m \u001b[39m    Used to hide legacy arguments that have been deprecated.\u001b[39;00m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=1094'>1095</a>\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=1095'>1096</a>\u001b[0m return_dict \u001b[39m=\u001b[39m return_dict \u001b[39mif\u001b[39;00m return_dict \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39melse\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mconfig\u001b[39m.\u001b[39muse_return_dict\n\u001b[0;32m-> <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=1097'>1098</a>\u001b[0m outputs \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mroberta(\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=1098'>1099</a>\u001b[0m     input_ids,\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=1099'>1100</a>\u001b[0m     attention_mask\u001b[39m=\u001b[39;49mattention_mask,\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=1100'>1101</a>\u001b[0m     token_type_ids\u001b[39m=\u001b[39;49mtoken_type_ids,\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=1101'>1102</a>\u001b[0m     position_ids\u001b[39m=\u001b[39;49mposition_ids,\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=1102'>1103</a>\u001b[0m     head_mask\u001b[39m=\u001b[39;49mhead_mask,\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=1103'>1104</a>\u001b[0m     inputs_embeds\u001b[39m=\u001b[39;49minputs_embeds,\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=1104'>1105</a>\u001b[0m     encoder_hidden_states\u001b[39m=\u001b[39;49mencoder_hidden_states,\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=1105'>1106</a>\u001b[0m     encoder_attention_mask\u001b[39m=\u001b[39;49mencoder_attention_mask,\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=1106'>1107</a>\u001b[0m     output_attentions\u001b[39m=\u001b[39;49moutput_attentions,\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=1107'>1108</a>\u001b[0m     output_hidden_states\u001b[39m=\u001b[39;49moutput_hidden_states,\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=1108'>1109</a>\u001b[0m     return_dict\u001b[39m=\u001b[39;49mreturn_dict,\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=1109'>1110</a>\u001b[0m )\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=1110'>1111</a>\u001b[0m sequence_output \u001b[39m=\u001b[39m outputs[\u001b[39m0\u001b[39m]\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=1111'>1112</a>\u001b[0m prediction_scores \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mlm_head(sequence_output)\n",
      "File \u001b[0;32m~/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py:1110\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1105'>1106</a>\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1106'>1107</a>\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1107'>1108</a>\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1108'>1109</a>\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1109'>1110</a>\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1110'>1111</a>\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1111'>1112</a>\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n",
      "File \u001b[0;32m~/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py:844\u001b[0m, in \u001b[0;36mRobertaModel.forward\u001b[0;34m(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m    <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=836'>837</a>\u001b[0m \u001b[39m# Prepare head mask if needed\u001b[39;00m\n\u001b[1;32m    <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=837'>838</a>\u001b[0m \u001b[39m# 1.0 in head_mask indicate we keep the head\u001b[39;00m\n\u001b[1;32m    <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=838'>839</a>\u001b[0m \u001b[39m# attention_probs has shape bsz x n_heads x N x N\u001b[39;00m\n\u001b[1;32m    <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=839'>840</a>\u001b[0m \u001b[39m# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]\u001b[39;00m\n\u001b[1;32m    <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=840'>841</a>\u001b[0m \u001b[39m# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]\u001b[39;00m\n\u001b[1;32m    <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=841'>842</a>\u001b[0m head_mask \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mget_head_mask(head_mask, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mconfig\u001b[39m.\u001b[39mnum_hidden_layers)\n\u001b[0;32m--> <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=843'>844</a>\u001b[0m embedding_output \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49membeddings(\n\u001b[1;32m    <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=844'>845</a>\u001b[0m     input_ids\u001b[39m=\u001b[39;49minput_ids,\n\u001b[1;32m    <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=845'>846</a>\u001b[0m     position_ids\u001b[39m=\u001b[39;49mposition_ids,\n\u001b[1;32m    <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=846'>847</a>\u001b[0m     token_type_ids\u001b[39m=\u001b[39;49mtoken_type_ids,\n\u001b[1;32m    <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=847'>848</a>\u001b[0m     inputs_embeds\u001b[39m=\u001b[39;49minputs_embeds,\n\u001b[1;32m    <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=848'>849</a>\u001b[0m     past_key_values_length\u001b[39m=\u001b[39;49mpast_key_values_length,\n\u001b[1;32m    <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=849'>850</a>\u001b[0m )\n\u001b[1;32m    <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=850'>851</a>\u001b[0m encoder_outputs \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mencoder(\n\u001b[1;32m    <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=851'>852</a>\u001b[0m     embedding_output,\n\u001b[1;32m    <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=852'>853</a>\u001b[0m     attention_mask\u001b[39m=\u001b[39mextended_attention_mask,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=860'>861</a>\u001b[0m     return_dict\u001b[39m=\u001b[39mreturn_dict,\n\u001b[1;32m    <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=861'>862</a>\u001b[0m )\n\u001b[1;32m    <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=862'>863</a>\u001b[0m sequence_output \u001b[39m=\u001b[39m encoder_outputs[\u001b[39m0\u001b[39m]\n",
      "File \u001b[0;32m~/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py:1110\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1105'>1106</a>\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1106'>1107</a>\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1107'>1108</a>\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1108'>1109</a>\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1109'>1110</a>\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1110'>1111</a>\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1111'>1112</a>\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n",
      "File \u001b[0;32m~/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py:132\u001b[0m, in \u001b[0;36mRobertaEmbeddings.forward\u001b[0;34m(self, input_ids, token_type_ids, position_ids, inputs_embeds, past_key_values_length)\u001b[0m\n\u001b[1;32m    <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=128'>129</a>\u001b[0m         token_type_ids \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mzeros(input_shape, dtype\u001b[39m=\u001b[39mtorch\u001b[39m.\u001b[39mlong, device\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mposition_ids\u001b[39m.\u001b[39mdevice)\n\u001b[1;32m    <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=130'>131</a>\u001b[0m \u001b[39mif\u001b[39;00m inputs_embeds \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m--> <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=131'>132</a>\u001b[0m     inputs_embeds \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mword_embeddings(input_ids)\n\u001b[1;32m    <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=132'>133</a>\u001b[0m token_type_embeddings \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtoken_type_embeddings(token_type_ids)\n\u001b[1;32m    <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/transformers/models/roberta/modeling_roberta.py?line=134'>135</a>\u001b[0m embeddings \u001b[39m=\u001b[39m inputs_embeds \u001b[39m+\u001b[39m token_type_embeddings\n",
      "File \u001b[0;32m~/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py:1110\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1105'>1106</a>\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1106'>1107</a>\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1107'>1108</a>\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1108'>1109</a>\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1109'>1110</a>\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1110'>1111</a>\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/module.py?line=1111'>1112</a>\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n",
      "File \u001b[0;32m~/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/sparse.py:158\u001b[0m, in \u001b[0;36mEmbedding.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/sparse.py?line=156'>157</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39minput\u001b[39m: Tensor) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m Tensor:\n\u001b[0;32m--> <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/sparse.py?line=157'>158</a>\u001b[0m     \u001b[39mreturn\u001b[39;00m F\u001b[39m.\u001b[39;49membedding(\n\u001b[1;32m    <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/sparse.py?line=158'>159</a>\u001b[0m         \u001b[39minput\u001b[39;49m, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mweight, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mpadding_idx, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmax_norm,\n\u001b[1;32m    <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/modules/sparse.py?line=159'>160</a>\u001b[0m         \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mnorm_type, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mscale_grad_by_freq, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msparse)\n",
      "File \u001b[0;32m~/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/functional.py:2183\u001b[0m, in \u001b[0;36membedding\u001b[0;34m(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)\u001b[0m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/functional.py?line=2176'>2177</a>\u001b[0m     \u001b[39m# Note [embedding_renorm set_grad_enabled]\u001b[39;00m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/functional.py?line=2177'>2178</a>\u001b[0m     \u001b[39m# XXX: equivalent to\u001b[39;00m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/functional.py?line=2178'>2179</a>\u001b[0m     \u001b[39m# with torch.no_grad():\u001b[39;00m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/functional.py?line=2179'>2180</a>\u001b[0m     \u001b[39m#   torch.embedding_renorm_\u001b[39;00m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/functional.py?line=2180'>2181</a>\u001b[0m     \u001b[39m# remove once script supports set_grad_enabled\u001b[39;00m\n\u001b[1;32m   <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/functional.py?line=2181'>2182</a>\u001b[0m     _no_grad_embedding_renorm_(weight, \u001b[39minput\u001b[39m, max_norm, norm_type)\n\u001b[0;32m-> <a href='file:///home/alfos/workspace/darpa/train_huggingface_from_scratch/env/lib/python3.9/site-packages/torch/nn/functional.py?line=2182'>2183</a>\u001b[0m \u001b[39mreturn\u001b[39;00m torch\u001b[39m.\u001b[39;49membedding(weight, \u001b[39minput\u001b[39;49m, padding_idx, scale_grad_by_freq, sparse)\n",
      "\u001b[0;31mRuntimeError\u001b[0m: CUDA error: device-side assert triggered"
     ]
    }
   ],
   "source": [
    "# Train the model\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "6264f85c0b37a06e4c227b113684058583ab955329524df471d3ea8b0cce5507"
  },
  "kernelspec": {
   "display_name": "Python 3.9.7 ('env': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}