Teja Gollapudi TejaGollapudi

## generate_alpaca_dataset.py
import json

import pandas as pd

with open('alpaca_data.json') as f:
    data = json.load(f)


new_format = []
for i, point in enumerate(data):

## lora_seq2seq.py
# Modified from https://github.com/huggingface/peft/blob/main/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py
import argparse
import gc
import logging
import os
import threading

import psutil
import torch
from accelerate import Accelerator

## template_config.yaml
compute_environment: LOCAL_MACHINE
deepspeed_config:
  gradient_accumulation_steps: 22
  offload_optimizer_device: cpu
  offload_param_device: cpu
  zero3_init_flag: true
  zero3_save_16bit_model: false
  zero_stage: 3
distributed_type: DEEPSPEED
downcast_bf16: 'no'

## merge_weights.py
# Modified from https://github.com/lvwerra/trl/blob/main/examples/sentiment/scripts/gpt-neox-20b_peft/merge_peft_adapter.py
import argparse
import os

import peft
import torch
from peft import PeftConfig, PeftModel
from transformers import AutoTokenizer, T5ForConditionalGeneration


## conda_env.yaml
name: lora_training
channels:
  - pytorch
  - conda-forge
dependencies:
  - cudatoolkit-dev=11.3.1
  - numpy=1.24.2
  - pip=23.0.1
  - python=3.8.16
  - pip:

## Demo.ipynb

      
              1 file
            
          
              0 forks
            
          
              0 comments
            
          
              1 star
            
          
                TejaGollapudi
                / Demo.ipynb
            
            
              Created
              June 30, 2022 04:16
            
              
                BERT pretraining demo notebook
              
          
        Loading

      Sorry, something went wrong. Reload?
      Sorry, we cannot display this file.
      Sorry, this file is invalid so it cannot be displayed.
      
          Viewer requires iframe.
	import json

	import pandas as pd

	with open('alpaca_data.json') as f:
	data = json.load(f)


	new_format = []
	for i, point in enumerate(data):
	# Modified from https://github.com/huggingface/peft/blob/main/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py
	import argparse
	import gc
	import logging
	import os
	import threading

	import psutil
	import torch
	from accelerate import Accelerator
	compute_environment: LOCAL_MACHINE
	deepspeed_config:
	gradient_accumulation_steps: 22
	offload_optimizer_device: cpu
	offload_param_device: cpu
	zero3_init_flag: true
	zero3_save_16bit_model: false
	zero_stage: 3
	distributed_type: DEEPSPEED
	downcast_bf16: 'no'
	# Modified from https://github.com/lvwerra/trl/blob/main/examples/sentiment/scripts/gpt-neox-20b_peft/merge_peft_adapter.py
	import argparse
	import os

	import peft
	import torch
	from peft import PeftConfig, PeftModel
	from transformers import AutoTokenizer, T5ForConditionalGeneration
	name: lora_training
	channels:
	- pytorch
	- conda-forge
	dependencies:
	- cudatoolkit-dev=11.3.1
	- numpy=1.24.2
	- pip=23.0.1
	- python=3.8.16
	- pip: