iohub

#!/bin/bash

# http://coldmooon.github.io/2015/08/10/fans_control/
speed=$1
echo $speed
export DISPLAY=:0.0
nvidia-settings -a "[gpu:0]/GPUFanControlState=1" -a "[fan:0]/GPUTargetFanSpeed="$speed
nvidia-settings -a "[gpu:0]/GPUFanControlState=1" -a "[fan:1]/GPUTargetFanSpeed="$speed

iohub

from typing import Union

from fastapi import FastAPI
from pydantic import BaseModel

import numpy as np
import torch
from utils import (DEFAULT_HF_MODEL_DIRS, DEFAULT_PROMPT_TEMPLATES,
                   load_tokenizer, read_model_name, throttle_generator)

iohub

import sys
import json
from typing import Union
from pathlib import Path


def _resolve_path(path: Union[str, Path]) -> Path:
    return Path(path).expanduser().resolve()

iohub

set OPENAI_API_KEY=xxx
python startup.py -a -i


def get_ChatOpenAI(
        model_name: str,
        temperature: float,
        max_tokens: int = None,
        streaming: bool = True,
        callbacks: List[Callable] = [],

iohub

Download UniDic /home/do/ssd/env/tts/lib/python3.10/site-packages/unidic/dicdir

iohub

import torch
from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer
import transformers
adapters_name = "/home/do/ssd/proj/finetune-hub/ck/checkpoint-1000"
model_name = "/home/do/ssd/modelhub/Starling-LM-7B-beta"


device = "cuda"  # the device to load the model onto

iohub

from transformers import TrainingArguments
from trl import SFTTrainer
from datasets import Dataset
from unsloth import FastMistralModel, FastLanguageModel, FastLlamaModel
import torch
import pandas as pd

'''
  {
    "instruction": "编辑一句话，改变句子结构，让它更加有趣。",

iohub

from unsloth import FastMistralModel, FastLanguageModel, FastLlamaModel
import torch


max_seq_length = 4096 # Can change to whatever number <= 4096
dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+
load_in_4bit = True # Use 4bit quantization to reduce memory usage. Can be False.
hf_token = "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"

model, tokenizer = FastMistralModel.from_pretrained(

iohub

docker pull nvcr.io/nvidia/tritonserver:23.10-trtllm-python-py3
docker run -dt --name tensorrt-llm  --restart=always  --gpus all  --network=host  --shm-size=4g  -m 64G  -v /home/:/home  -w /home/do nvcr.io/nvidia/tritonserver:23.10-trtllm-python-py3 /bin/bash
docker exec -it trtllm /bin/bash


git clone https://github.com/NVIDIA/TensorRT-LLM.git
cd TensorRT-LLM
git submodule update --init --recursive

python3 ./scripts/build_wheel.py --trt_root /usr/local/tensorrt

iohub

import time
import os

# delete it
class FakeWB:
    def __init__(self, name, age):
        self.name = name
        self.age = age