ebsmothers
ebsmothers
/ gist:ce0aeb128115866af42e692d678e5193
Created
June 4, 2024 01:34
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from torch import nn | |
| from torchtune.utils import get_memory_stats, get_device | |
| from torchao.dtypes.nf4tensor import to_nf4 | |
| from bitsandbytes.functional import quantize_nf4 | |
| def main(): | |
| device = get_device('cuda') | |
| # Size of Llama3-8B output projection weight |
ebsmothers
/ gist:125c822a9cb0323a2ebfe59dae54abc3
Created
March 15, 2024 19:15
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import torch | |
| from transformers import ( | |
| AutoModelForCausalLM, | |
| AutoTokenizer, | |
| BitsAndBytesConfig, | |
| HfArgumentParser, | |
| TrainingArguments, | |
| pipeline, | |
| logging, | |
| GenerationConfig, |
ebsmothers
/ gist:ea631a8f0857d79ece5245c8ab1e8e52
Created
February 8, 2024 00:04
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| def validate_state_dict_for_lora( | |
| *, | |
| lora_modules: List[str], | |
| full_model_state_dict_keys: List[str], | |
| lora_state_dict_keys: Optional[List[str]] = None, | |
| base_model_state_dict_keys: Optional[List[str]] = None, | |
| ): | |
| is_lora_param = lambda x: "lora" in x and any([k in x for k in lora_modules]) | |
| for k in full_model_state_dict_keys: | |
| if not is_lora_param(k): |
ebsmothers
/ gist:e7c616cb8d2fb5aea4186ae32a1fb22d
Created
October 23, 2023 18:53
FLAVA load weights
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| _IncompatibleKeys(missing_keys=['classifier.model.0.weight', 'classifier.model.0.bias', 'classifier.model.3.weight', 'classifier.model.3.bias'], unexpected_keys=['image_codebook.encoder.blocks.input.w', 'image_codebook.encoder.blocks.input.b', 'image_codebook.encoder.blocks.group_1.block_1.res_path.conv_1.w', 'image_codebook.encoder.blocks.group_1.block_1.res_path.conv_1.b', 'image_codebook.encoder.blocks.group_1.block_1.res_path.conv_2.w', 'image_codebook.encoder.blocks.group_1.block_1.res_path.conv_2.b', 'image_codebook.encoder.blocks.group_1.block_1.res_path.conv_3.w', 'image_codebook.encoder.blocks.group_1.block_1.res_path.conv_3.b', 'image_codebook.encoder.blocks.group_1.block_1.res_path.conv_4.w', 'image_codebook.encoder.blocks.group_1.block_1.res_path.conv_4.b', 'image_codebook.encoder.blocks.group_1.block_2.res_path.conv_1.w', 'image_codebook.encoder.blocks.group_1.block_2.res_path.conv_1.b', 'image_codebook.encoder.blocks.group_1.block_2.res_path.conv_2.w', 'image_codebook.encoder.blocks.group_1.bloc |
ebsmothers
/ gist:03ea9728df2e905a1ad52b1ca193f6c8
Created
October 23, 2023 18:53
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| _IncompatibleKeys(missing_keys=['classifier.model.0.weight', 'classifier.model.0.bias', 'classifier.model.3.weight', 'classifier.model.3.bias'], unexpected_keys=['image_codebook.encoder.blocks.input.w', 'image_codebook.encoder.blocks.input.b', 'image_codebook.encoder.blocks.group_1.block_1.res_path.conv_1.w', 'image_codebook.encoder.blocks.group_1.block_1.res_path.conv_1.b', 'image_codebook.encoder.blocks.group_1.block_1.res_path.conv_2.w', 'image_codebook.encoder.blocks.group_1.block_1.res_path.conv_2.b', 'image_codebook.encoder.blocks.group_1.block_1.res_path.conv_3.w', 'image_codebook.encoder.blocks.group_1.block_1.res_path.conv_3.b', 'image_codebook.encoder.blocks.group_1.block_1.res_path.conv_4.w', 'image_codebook.encoder.blocks.group_1.block_1.res_path.conv_4.b', 'image_codebook.encoder.blocks.group_1.block_2.res_path.conv_1.w', 'image_codebook.encoder.blocks.group_1.block_2.res_path.conv_1.b', 'image_codebook.encoder.blocks.group_1.block_2.res_path.conv_2.w', 'image_codebook.encoder.blocks.group_1.bloc |