amrakm

import tiktoken

tokenizer = tiktoken.get_encoding('p50k_base')

# create the length function
def tiktoken_len(text):
    tokens = tokenizer.encode(
        text,
        disallowed_special=()
    )

amrakm

from langchain.text_splitter import RecursiveCharacterTextSplitter

text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=500,
    chunk_overlap=20,
    length_function=tiktoken_len,
    separators=["\n\n", "\n", " ", ""]
)

amrakm

import json 

dict = json.loads(response.text)

print(json.dumps(dict, indent=2))

amrakm

import pandas as pd



import firebase_admin
from firebase_admin import credentials
from firebase_admin import firestore

cred = credentials.Certificate("XXX.json")
firebase_admin.initialize_app(cred)

amrakm

from PIL import Image


def expand_tile_pil_img(pil_img):

    blank_image = Image.new("RGB", (pil_img.size[0] * 2, pil_img.size[1] * 2))

    blank_image.paste(pil_img, (0,0))
    blank_image.paste(pil_img, (pil_img.size[0], 0))
    blank_image.paste(pil_img, (0, pil_img.size[1]))

amrakm

import torch
from diffusers import StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", use_auth_token=True).to("cuda")


seed= 111111
height =512
width = 512
device = 'cuda'

amrakm

jupyter notebook \ 
    --NotebookApp.allow_origin='https://colab.research.google.com' \ 
    --port=8888 \ 
    --NotebookApp.port_retries=0

amrakm

#@markdown **NVIDIA GPU**
import subprocess
sub_p_res = subprocess.run(['nvidia-smi', '--query-gpu=name,memory.total,memory.free', '--format=csv,noheader'], stdout=subprocess.PIPE).stdout.decode('utf-8')
print(sub_p_res)

amrakm

# split reviews into multiple parts based on max_token size, appending one sentence at a time until the part hits the max token limit

def split_rev(rev, max_tokens = 384):


    rev_sentences = rev.split('.')


    parts_list =  []

amrakm

import pip


def import_or_install(package):
    try:
        __import__(package)
    except ImportError:
        pip.main(["install", package])