Created
July 15, 2024 00:18
-
-
Save joelburget/81ce38c42264655bf8b2fcfa2497a239 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"cells": [ | |
{ | |
"cell_type": "code", | |
"execution_count": 12, | |
"id": "b0374368-eb08-4ceb-8c8e-c8f409aa7213", | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"import torch\n", | |
"import torch.nn as nn\n", | |
"from huggingface_hub import PyTorchModelHubMixin\n", | |
"from transformers import AutoConfig, AutoModel\n", | |
"from transformers.models.mixtral.modeling_mixtral import MixtralDecoderLayer\n", | |
"from transformers.models.gpt_neo.modeling_gpt_neo import GPTNeoBlock, GPTNeoModel" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 2, | |
"id": "94c38ffa-afbf-4056-a0f0-9015a3e18fc9", | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"resolved_archive_file='/Users/joel/.cache/huggingface/hub/models--EleutherAI--gpt-neo-125M/snapshots/21def0189f5705e2521767faed922f1f15e7d7db/model.safetensors'\n", | |
"f=<builtins.safe_open object at 0x1677db7b0>\n" | |
] | |
} | |
], | |
"source": [ | |
"hf_model = AutoModel.from_pretrained(\"EleutherAI/gpt-neo-125M\")" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 19, | |
"id": "defef29c-671a-43af-861a-c64eda75ec8c", | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"class MyModel(GPTNeoModel):\n", | |
" def __init__(self, config):\n", | |
" super().__init__(config)\n", | |
" self.h = nn.ModuleList([GPTNeoBlock(config, 0)])" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 23, | |
"id": "f313e455-503d-4c36-b736-d3154b136f50", | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"data": { | |
"application/vnd.jupyter.widget-view+json": { | |
"model_id": "6dd45c59fcbd4f0ba1fc003f558b4e6e", | |
"version_major": 2, | |
"version_minor": 0 | |
}, | |
"text/plain": [ | |
"model.safetensors: 0%| | 0.00/189M [00:00<?, ?B/s]" | |
] | |
}, | |
"metadata": {}, | |
"output_type": "display_data" | |
}, | |
{ | |
"data": { | |
"text/plain": [ | |
"CommitInfo(commit_url='https://huggingface.co/joelb/my-awesome-model/commit/f1090d0a00028679dedc0a4efea9d11f434d5556', commit_message='Upload model', commit_description='', oid='f1090d0a00028679dedc0a4efea9d11f434d5556', pr_url=None, pr_revision=None, pr_num=None)" | |
] | |
}, | |
"execution_count": 23, | |
"metadata": {}, | |
"output_type": "execute_result" | |
} | |
], | |
"source": [ | |
"config = AutoConfig.from_pretrained(\"EleutherAI/gpt-neo-125M\")\n", | |
"config.num_layers = 1\n", | |
"config.attention_layers = config.attention_layers[:1]\n", | |
"config.attention_types = [[['global'], 1]]\n", | |
"model = MyModel(config)\n", | |
"model.push_to_hub(\"joelb/my-awesome-model\", config=config)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 24, | |
"id": "dc5a44cd-a4ac-4c70-a4a9-a04b57e5b189", | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"data": { | |
"application/vnd.jupyter.widget-view+json": { | |
"model_id": "d3c7daec4a0743e89ed2cce9c132f409", | |
"version_major": 2, | |
"version_minor": 0 | |
}, | |
"text/plain": [ | |
"config.json: 0%| | 0.00/930 [00:00<?, ?B/s]" | |
] | |
}, | |
"metadata": {}, | |
"output_type": "display_data" | |
}, | |
{ | |
"data": { | |
"application/vnd.jupyter.widget-view+json": { | |
"model_id": "33fd4573a95e450a94f5586d5aa3f27f", | |
"version_major": 2, | |
"version_minor": 0 | |
}, | |
"text/plain": [ | |
"model.safetensors: 0%| | 0.00/189M [00:00<?, ?B/s]" | |
] | |
}, | |
"metadata": {}, | |
"output_type": "display_data" | |
}, | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"resolved_archive_file='/Users/joel/.cache/huggingface/hub/models--joelb--my-awesome-model/snapshots/f1090d0a00028679dedc0a4efea9d11f434d5556/model.safetensors'\n", | |
"f=<builtins.safe_open object at 0x169b09bf0>\n" | |
] | |
} | |
], | |
"source": [ | |
"model2 = AutoModel.from_pretrained(\"joelb/my-awesome-model\")" | |
] | |
} | |
], | |
"metadata": { | |
"kernelspec": { | |
"display_name": "Python 3 (ipykernel)", | |
"language": "python", | |
"name": "python3" | |
}, | |
"language_info": { | |
"codemirror_mode": { | |
"name": "ipython", | |
"version": 3 | |
}, | |
"file_extension": ".py", | |
"mimetype": "text/x-python", | |
"name": "python", | |
"nbconvert_exporter": "python", | |
"pygments_lexer": "ipython3", | |
"version": "3.11.9" | |
} | |
}, | |
"nbformat": 4, | |
"nbformat_minor": 5 | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment