joshreini1

honest_feedbacks = [qa_relevance, qs_relevance, f_embed_dist, f_groundedness]

harmless_feedbacks = [f_controversiality, f_criminality, f_harmfulness, f_insensitivity,
                      f_maliciousness, f_misogyny, f_stereotypes, f_hate,
                      f_hatethreatening, f_violent, f_violentgraphic, f_selfharm]

helpful_feedbacks = [f_langmatch, f_conciseness]

feedback_suite = honest_feedbacks + harmless_feedbacks + helpful_feedbacks

joshreini1

claude_2  = LiteLLM(model_engine="claude-2")

context_relevance = (
    Feedback(claude_2.qs_relevance_with_cot_reasons,name = "Context Relevance")
    .on_input()
    .on(TruLlama.select_source_nodes().node.text)
    .aggregate(np.mean)
)

joshreini1

recorder = TruChain(chain, feedbacks=[relevance])

joshreini1

from trulens_eval import LiteLLM
litellm_provider  = LiteLLM(model_engine="gpt-3.5-turbo") # choose any model here!

# Define a relevance function using LiteLLM
relevance = Feedback(litellm_provider.relevance).on_input_output()

joshreini1

tru_query_engine = TruLlama(query_engine,
  app_id=f"My first RAG",
  feedbacks=[f_groundedness, f_qa_relevance, f_context_relevance])

joshreini1

# Question/answer relevance between overall question and answer.
f_qa_relevance = Feedback(openai_gpt4.relevance_with_cot_reasons, name = "Answer Relevance").on_input_output()

joshreini1

from langchain.chat_models import ChatOpenAI
from langchain.chains import RetrievalQA

# completion llm
llm = ChatOpenAI(
    model_name='gpt-3.5-turbo',
    temperature=0.0
)

qa = RetrievalQA.from_chain_type(

joshreini1

background_split = background_split.rename(columns={'oss_attr':'os_shap_influence',
  'is_mobile_attr':'is_mobile_shap_influence',
  'pageviews_attr':'pageviews_shap_influence',
  'country_attr':'country_shap_influence'})

joshreini1

# create (empty) virtual model
model_name = "BQML Virtual Model"
tru.add_model(model_name)

joshreini1

tru.add_data_collection("Visitor Data")