Michael Pieler MicPie

## HF_tokenizer.ipynb

      
              1 file
            
          
              0 forks
            
          
              0 comments
            
          
              0 stars
            
          
                MicPie
                / HF_tokenizer.ipynb
            
            
              Last active
              April 6, 2023 13:54
            
          
      Sorry, something went wrong. Reload?
      Sorry, we cannot display this file.
      Sorry, this file is invalid so it cannot be displayed.
      
          Viewer requires iframe.
      
    
## csv2jsonl.py
import argparse
import pandas as pd
import json


def main():
    parser = argparse.ArgumentParser(description="List the content of a folder")
    parser.add_argument(
            "path",
            metavar="path",

## resharded_to_hf_conversion.py
import numpy as np
import torch
import pdb


def take_out(flat_params, shape):
    return flat_params[:np.prod(shape)].view(*shape), flat_params[np.prod(shape):]


def get_slice(shard_size, shard_i):

## opt-175b_hf-config.json
{
  "_name_or_path": "./",
  "_remove_final_layer_norm": false,
  "activation_dropout": 0.0,
  "activation_function": "relu",
  "architectures": [
    "OPTForCausalLM"
  ],
  "attention_dropout": 0.0,
  "bos_token_id": 2,
	import argparse
	import pandas as pd
	import json


	def main():
	parser = argparse.ArgumentParser(description="List the content of a folder")
	parser.add_argument(
	"path",
	metavar="path",
	import numpy as np
	import torch
	import pdb


	def take_out(flat_params, shape):
	return flat_params[:np.prod(shape)].view(*shape), flat_params[np.prod(shape):]


	def get_slice(shard_size, shard_i):
	{
	"_name_or_path": "./",
	"_remove_final_layer_norm": false,
	"activation_dropout": 0.0,
	"activation_function": "relu",
	"architectures": [
	"OPTForCausalLM"
	],
	"attention_dropout": 0.0,
	"bos_token_id": 2,