Tested on NVIDIA RTX 4090, but these instructions also cover AMD and Mac in case you wanna try those.
This guide assumes you are running Linux (I ran this on Ubuntu).
Before you get excited:
import tiktoken | |
import langdetect | |
T = tiktoken.get_encoding("o200k_base") | |
length_dict = {} | |
for i in range(T.n_vocab): | |
try: | |
length_dict[i] = len(T.decode([i])) | |
except: |
from transformers import ( | |
AutoConfig, | |
AutoTokenizer, | |
BitsAndBytesConfig, | |
GenerationConfig, | |
AutoModelForCausalLM, | |
LlamaTokenizerFast, | |
PreTrainedModel, | |
TextIteratorStreamer, | |
StoppingCriteria, |
# coding=utf-8 | |
# Copyright 2023 The HuggingFace Inc. team. All rights reserved. | |
# | |
# Licensed under the Apache License, Version 2.0 (the "License"); | |
# you may not use this file except in compliance with the License. | |
# You may obtain a copy of the License at | |
# | |
# http://www.apache.org/licenses/LICENSE-2.0 | |
# | |
# Unless required by applicable law or agreed to in writing, software |