mtisz

import subprocess as sp
import logging
from abc import ABC
import time

from concurrent.futures import ThreadPoolExecutor

"""
Your terminal command goes here
"""

mtisz

Fork of qlora: https://github.com/jondurbin/qlora

Make sure to change dataset format, and dataset path to your file, along with model/output paths.

If you want to modify the prompt format, edit this: https://github.com/jondurbin/qlora/blob/main/qlora.py#L433

Args used:

python qlora.py \

mtisz

import numpy as np
import torch
import jax
from tqdm import tqdm

from model import LanguageModelConfig, TransformerConfig, QuantizedWeight8bit as QW8Bit
from runners import InferenceRunner, ModelRunner, sample_from_model

CKPT_PATH = "./checkpoints"

mtisz

---
version: "2.0"
services:
  service-1:
    image: winglian/axolotl:main-py3.11-cu121-2.2.1
    expose:
      - port: 80
        as: 80
        to:
          - global: true

mtisz

base_model: mistral-community/Mixtral-8x22B-v0.1
model_type: MixtralForCausalLM
tokenizer_type: AutoTokenizer
is_mistral_derived_model: false
trust_remote_code: true

load_in_8bit: false
load_in_4bit: true
strict: false

mtisz

base_model: meta-llama/Meta-Llama-3-70B
model_type: LlamaForCausalLM
tokenizer_type: AutoTokenizer

load_in_8bit: false
load_in_4bit: true
strict: false

datasets:
  - path: /home/migel/ai_datasets/tess-v1.5b-chatml.jsonl

mtisz

from transformers import AutoTokenizer, AutoModelForCausalLM
import torch

# model_path = "/home/migel/gemma-2-27b"
model_path = "google/gemma-2-27b-it"

tokenizer = AutoTokenizer.from_pretrained(model_path)
model = AutoModelForCausalLM.from_pretrained(
    model_path, 
    device_map="auto",