alfredplpl

# MIT License
# This code will run on VRAM 12GB+ GPU such as T4, RTX 3060
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.document_loaders import PyPDFLoader
from langchain.vectorstores import FAISS
from langchain.chains import RetrievalQA
from langchain.embeddings import HuggingFaceEmbeddings
from langchain.llms.huggingface_pipeline import HuggingFacePipeline

alfredplpl

# MIT License
from transformers import AutoTokenizer
import transformers
from langchain.document_loaders import PyPDFLoader
import torch

model = "NousResearch/Yarn-Llama-2-13b-128k"

tokenizer = AutoTokenizer.from_pretrained(model)
pipeline = transformers.pipeline(

alfredplpl

from datasets import load_dataset
import requests
from PIL import Image
from tqdm import tqdm

dataset = load_dataset("laion/dalle-3-dataset",split="train")


for i,row in enumerate(tqdm(dataset)):
    with open(f"dalle3/{i:06}.txt","w") as f:

alfredplpl

alfredplpl

from diffusers import DiffusionPipeline
import torch
from consistencydecoder import ConsistencyDecoder
from PIL import Image
import numpy as np

pipe = DiffusionPipeline.from_pretrained("SimianLuo/LCM_Dreamshaper_v7", torch_dtype=torch.float32)

decoder_consistency = ConsistencyDecoder(device="cuda:0") # Model size: 2.49 GB

alfredplpl

# MIT
from diffusers import StableDiffusionXLPipeline
import torch

pipe = StableDiffusionXLPipeline.from_single_file('/path/to/checkpoint.safetensors', torch_dtype=torch.float16)
pipe.save_pretrained('/path/to/diffusers_version')

alfredplpl

# Reference #1: https://note.com/npaka/n/nc55e44e407ff
# Reference #2: https://huggingface.co/blog/gemma-peft
# Licence: MIT

from peft import LoraConfig

lora_config = LoraConfig(
    r=8,
    target_modules=["q_proj", "o_proj", "k_proj", "v_proj", "gate_proj", "up_proj", "down_proj"],
    task_type="CAUSAL_LM",

alfredplpl

import torch
from transformers import AutoTokenizer,AutoModelForCausalLM
import pandas

model_name_or_path = "cyberagent/calm2-7b-chat"
tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
model = AutoModelForCausalLM.from_pretrained(model_name_or_path, device_map="cpu", torch_dtype=torch.float32)

# https://github.com/nlp-waseda/JMMLU/blob/main/JMMLU/college_computer_science.csv
df=pandas.read_csv("college_computer_science.csv",header=None)