import requests
import html
import json
# Define the Texgen API endpoint
HOST = 'cmlllm-textgenuiurl'
URI = f'https://{HOST}/api/v1/chat'
# Function to send a message to Texgen
def send_message(user_input, history, username, continueFlag):
request = {
'user_input': user_input,
'max_new_tokens': 2500,
'auto_max_new_tokens': False,
'max_tokens_second': 0,
'history': history,
'mode': 'instruct', # Valid options: 'chat', 'chat-instruct', 'instruct'
'character': 'Example',
'instruction_template': 'Vicuna-v1.1', # Will get autodetected if unset
'your_name': username,
# 'name1': 'name of user', # Optional
# 'name2': 'name of character', # Optional
# 'context': 'character context', # Optional
# 'greeting': 'greeting', # Optional
# 'name1_instruct': 'You', # Optional
# 'name2_instruct': 'Assistant', # Optional
# 'context_instruct': 'context_instruct', # Optional
# 'turn_template': 'turn_template', # Optional
'regenerate': False,
'_continue': continueFlag,
# 'chat_instruct_command': 'Continue the chat dialogue below. Write a single reply for the character "<|character|>".\n\n<|prompt|>',
# Generation params. If 'preset' is set to different than 'None', the values
# in presets/preset-name.yaml are used instead of the individual numbers.
'preset': 'None',
'do_sample': True,
'temperature': 0.7,
'top_p': 0.1,
'typical_p': 1,
'epsilon_cutoff': 0, # In units of 1e-4
'eta_cutoff': 0, # In units of 1e-4
'tfs': 1,
'top_a': 0,
'repetition_penalty': 1.18,
'repetition_penalty_range': 0,
'top_k': 40,
'min_length': 0,
'no_repeat_ngram_size': 0,
'num_beams': 1,
'penalty_alpha': 0,
'length_penalty': 1,
'early_stopping': False,
'mirostat_mode': 0,
'mirostat_tau': 5,
'mirostat_eta': 0.1,
'grammar_string': '',
'guidance_scale': 1,
'negative_prompt': '',
'seed': -1,
'add_bos_token': True,
'truncation_length': 2048 * 5,
'ban_eos_token': False,
'custom_token_bans': '',
'skip_special_tokens': True,
'stopping_strings': []
}
response = requests.post(URI, json=request)
return response;
# Function to interact with the chatbot
def chat_with_bot():
chatUsername = input("Please enter your name :")
print(f'Hello {chatUsername}, Welcome to the Texgen Chatbot! \nType exit to quit')
# Basic example
user_input = ""
history = {'internal': [], 'visible': []}
continueGeneration = 'n'
while True:
if continueGeneration.lower() == 'n':
user_input = input(f'{chatUsername}:')
if user_input.lower() == "exit":
print("Goodbye!")
break
continueFlag = False if (continueGeneration.lower() == 'n') else True
response = send_message(user_input, history, chatUsername,continueFlag)
if response.status_code == 200:
data = response.json()
if data['results'][0]['history']['visible']:
history = data['results'][0]['history']
visible_response = history['visible'][-1][1]
unescaped_string = html.unescape(visible_response)
print(f'TextGen Bot: {unescaped_string}')
#If the response ends with a '?', user shoud answer that question first.
if not unescaped_string.endswith("?"):
# Assume the response is partial , and ask user to contuinue with the text generation.
continueGeneration = input("Do you want to continue generation of the current response (y/n)?")
continue
else:
print("TextGen Bot: Response is empty.")
else:
print("Error: Unable to get a response from the Texgen API.")
continueGeneration = 'n'
if __name__ == "__main__":
chat_with_bot()
Last active
October 11, 2023 21:58
-
-
Save vinodkc/a38447d770b72aaa6e7fce580bf532cd to your computer and use it in GitHub Desktop.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment