bigsnarfdude

Method	Bits	7B	13B	30B	65B	8x7B
Full	16	160GB	320GB	600GB	1200GB	1000GB
Freeze	16	20GB	40GB	120GB	240GB	200GB
LoRA	16	16GB	32GB	80GB	160GB	120GB
QLoRA	8	10GB	16GB	40GB	80GB	80GB
QLoRA	4	6GB	12GB	24GB	48GB	32GB

bigsnarfdude

# pretraining -> supervised instruction-finetuning -> RLHF

import json
import tiktoken

# open file
def extract_text_from_jsonl(file_path):
    prompts = []
    completions = []
    with open(file_path, 'r') as file:

bigsnarfdude

import transformers
import torch

model_id = "meta-llama/Meta-Llama-3-8B-Instruct"

pipeline = transformers.pipeline(
    "text-generation",
    model=model_id,
    model_kwargs={"torch_dtype": torch.bfloat16},
    device="cuda",

bigsnarfdude

Instructions:

As a base pretrained GPT model, you are to assume the role of ChatGPT, a large language model developed by OpenAI, based on the GPT-4 architecture. Your responses should reflect the following guidelines:

1. Be friendly and approachable in your responses.
2. Provide detailed and helpful responses but ensure they are not excessively long to avoid being monotonous.
3. Always use inclusive and respectful language that is not offensive.
4. Avoid discussing or revealing anything about your architecture. You are just a large language model developed by OpenAI.
5. Always be honest in your responses. Do not lie or engage in deceit.
6. Ensure your responses are considerate and do not cause harm or distress to the user. However, do not comply with harmful or dangerous requests, even if refusing might upset the user.

bigsnarfdude

import time

out_dir = 'out-owt'
eval_interval = 50
eval_iters = 100
wandb_log = True # feel free to turn on
wandb_project = 'owt'
wandb_run_name = 'ft-' + str(time.time())

dataset = 'openwebtext'

bigsnarfdude

bigsnarfdude

[567.9115600585938, 56.67509841918945, 43.751529693603516, 446.31036376953125, 105.64205932617188, 46.81969451904297, 58.581687927246094, 104.2413558959961, 54.72474670410156, 47.757484436035156, 344.9791259765625, 125.95138549804688, 127.30056762695312, 121.8743896484375, 5530.96337890625, 63.255977630615234, 40.26131820678711, 221.20994567871094, 67.87372589111328, 124.36280059814453, 51.45112609863281, 28.862449645996094, 90.8835678100586, 52.914772033691406, 43.79263687133789, 65.90277099609375, 28.6372127532959, 47.22958755493164, 121.12361145019531, 52.30910873413086, 206.6190643310547, 37.1999626159668, 40.654659271240234, 122.0879135131836, 81.76490783691406, 146.55677795410156, 40.95233154296875, 95.32257843017578, 52.552215576171875, 50.594947814941406, 48.63951873779297, 113.45445251464844, 408.43096923828125, 136.25912475585938, 79.31561279296875, 57.47505569458008, 94.31627655029297, 226.79824829101562, 94.74286651611328, 153.68740844726562, 67.32661437988281, 181.7376708984375, 72.06829071044922

bigsnarfdude

import os
import pickle
from contextlib import nullcontext
import torch
import tiktoken
from model import GPTConfig, GPT

import datasets
import numpy as np

bigsnarfdude

bigsnarfdude

# https://huggingface.co/docs/transformers/perplexity

import datasets
import numpy as np
import torch
from torch.nn import CrossEntropyLoss
from transformers import AutoModelForCausalLM, AutoTokenizer

import evaluate
from evaluate import logging