Dr. Q quinsulon

## smallberta_pretraining.ipynb

      
              1 file
            
          
              24 forks
            
          
              22 comments
            
          
              78 stars
            
          
                aditya-malte
                / smallberta_pretraining.ipynb
            
            
              Created
              February 22, 2020 13:41
            
              
                smallBERTa_Pretraining.ipynb
              
          
      Sorry, something went wrong. Reload?
      Sorry, we cannot display this file.
      Sorry, this file is invalid so it cannot be displayed.
      
          Viewer requires iframe.
      
    
## transformer-pytorch.py
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn import TransformerEncoder, TransformerEncoderLayer


class TransformerModel(nn.Module):

    def __init__(self, ntoken, ninp, nhead, nhid, nlayers, dropout=0.5):

## loading_wikipedia.py
import os; import psutil; import timeit
from datasets import load_dataset

mem_before = psutil.Process(os.getpid()).memory_info().rss >> 20
wiki = load_dataset("wikipedia", "20200501.en", split='train')
mem_after = psutil.Process(os.getpid()).memory_info().rss >> 20
print(f"RAM memory used: {(mem_after - mem_before)} MB")

s = """batch_size = 1000
for i in range(0, len(wiki), batch_size):
	import math
	import torch
	import torch.nn as nn
	import torch.nn.functional as F
	from torch.nn import TransformerEncoder, TransformerEncoderLayer


	class TransformerModel(nn.Module):

	def __init__(self, ntoken, ninp, nhead, nhid, nlayers, dropout=0.5):
	import os; import psutil; import timeit
	from datasets import load_dataset

	mem_before = psutil.Process(os.getpid()).memory_info().rss >> 20
	wiki = load_dataset("wikipedia", "20200501.en", split='train')
	mem_after = psutil.Process(os.getpid()).memory_info().rss >> 20
	print(f"RAM memory used: {(mem_after - mem_before)} MB")

	s = """batch_size = 1000
	for i in range(0, len(wiki), batch_size):