ritun16

from openai import OpenAI

# init client and connect to localhost server
client = OpenAI(
    api_key="<generated_api_key>",
    base_url="https://3511-82-713-192-232.ngrok-free.app/" # ngrok prublic URL / Localhost IP:PORT
)

# call API
chat_completion = client.chat.completions.create(

ritun16

import nest_asyncio
from pyngrok import ngrok, conf
import uvicorn

conf.get_default().auth_token = userdata.get('NGROK_AUTH_TOKEN')
ngrok_tunnel = ngrok.connect(8000)
print('Public URL:', ngrok_tunnel.public_url)
nest_asyncio.apply() # IF you are running on colab / notebook
uvicorn.run(app, port=8000)

ritun16

app = FastAPI(title="OpenAI-compatible API")

llm_server = vLLMMetaLlama3_1()

@app.post("/chat/completions")
@rate_limit()
async def chat_completions(request: ChatCompletionRequest, authorization: str = Header(...)):
    response_content = llm_server(temperature=request.temperature, top_p=request.top_p, max_tokens=request.max_tokens, messages=request.messages)

    return {

ritun16

import hashlib
import time
from dataclasses import dataclass
from functools import wraps
from typing import Any, Callable
import sqlite3

def rate_limit():
    def decorator(func: Callable[[Request], Any]) -> Callable[[Request], Any]:
        usage: dict[str, list[float]] = {}

ritun16

from vllm import LLM, SamplingParams
from transformers import AutoTokenizer

class vLLMMetaLlama3_1():

  def __init__(self):
    # Replace with other LLMs of your choice
    model_id = "hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4"
    self.llm = LLM(model=model_id) # Setup serving LLM
    self.tokenizer = AutoTokenizer.from_pretrained(model_id) # Set up tokenizer

ritun16

class ChatMessage(BaseModel):
    role: str
    content: str

class ChatCompletionRequest(BaseModel):
    model: str = "mock-gpt-model"
    messages: List[ChatMessage]
    max_tokens: Optional[int] = 512
    temperature: Optional[float] = 0.1
    top_p: Optional[float] = 0.9

ritun16

import streamlit as st
import sqlite3
import random
import string
from datetime import datetime, timedelta

# Function to generate a 32-character alphanumeric API key
def generate_api_key():
    return 'sk-user-' + ''.join(random.choices(string.ascii_letters + string.digits, k=32))

ritun16

def log_in_mechanism():
    email = st.text_input("Enter your email:")
    if st.button("Submit Email"):
        with st.spinner('Verifying and sending you email'):
            if email:
                email_strip = email.strip(" \n")
                if is_supported_domain(email_strip):
                    otp_result = get_otp_and_exipry(email_strip)
                    current_time = datetime.now()
                    if otp_result and datetime.strptime(otp_result[1], '%Y-%m-%d %H:%M:%S.%f') > current_time:

ritun16

# Users Table
'''CREATE TABLE IF NOT EXISTS users (
                    email TEXT PRIMARY KEY,
                    otp TEXT,
                    otp_expiry TIMESTAMP
                    )
'''

# API Keys Table
'''CREATE TABLE IF NOT EXISTS api_keys (

ritun16

'''
pip install azure-core
pip install azure-communication-email
'''

import os
import sys
import time
import re
import json