x

app.py

@@ -9,7 +9,6 @@ st.set_page_config(page_title="Aidan Bench - Generator")
 
 st.title("Aidan Bench - Generator")
 
-
 # API Key Inputs with Security and User Experience Enhancements
 st.warning("Please keep your API keys secure and confidential. This app does not store or log your API keys.")
 
@@ -94,9 +93,13 @@ if st.session_state.open_router_key and st.session_state.openai_api_key:
         st.session_state.user_questions = []
 
     # Threshold Sliders
-    st.subheader("Threshold Sliders")
-    coherence_threshold = st.slider("Coherence Threshold (0-5):", 0, 5, 3)
-    novelty_threshold = st.slider("Novelty Threshold (0-1):", 0.0, 1.0, 0.1)
+    st.sidebar.subheader("Threshold Sliders")
+    coherence_threshold = st.sidebar.slider("Coherence Threshold (0-5):", 0, 5, 3)
+    novelty_threshold = st.sidebar.slider("Novelty Threshold (0-1):", 0.0, 1.0, 0.1)
+
+    st.sidebar.subheader("Temp Sliders")
+    temp_threshold = st.sidebar.slider("Temperature (0-2):", 0.0, 2.0, 1.0)
+    top_p = st.sidebar.slider("Top P (0-1):", 0.0, 1.0, 1.0)
 
     # Workflow Selection
     workflow = st.radio("Select Workflow:", ["Use Predefined Questions", "Use User-Defined Questions"])
@@ -159,9 +162,9 @@ if st.session_state.open_router_key and st.session_state.openai_api_key:
 
             # Benchmarking logic using the chosen execution mode
             if execution_mode == "Sequential":
-                question_results = benchmark_model_sequential(model_name, selected_questions, st.session_state.open_router_key, st.session_state.openai_api_key,judge_model_name,coherence_threshold,novelty_threshold)
+                question_results = benchmark_model_sequential(model_name, selected_questions, st.session_state.open_router_key, st.session_state.openai_api_key,judge_model_name,coherence_threshold,novelty_threshold,temp_threshold,top_p)
             else:  # Multithreaded
-                question_results = benchmark_model_multithreaded(model_name, selected_questions, st.session_state.open_router_key, st.session_state.openai_api_key, max_threads, judge_model_name, coherence_threshold,novelty_threshold)
+                question_results = benchmark_model_multithreaded(model_name, selected_questions, st.session_state.open_router_key, st.session_state.openai_api_key, max_threads, judge_model_name, coherence_threshold,novelty_threshold,temp_threshold,top_p)
 
             results.extend(question_results)
 

main.py

@@ -7,25 +7,25 @@ import threading
 import streamlit as st  # Import Streamlit
 import queue
 
-def generate_answer(question, previous_answers, model_name, open_router_key, openai_api_key):
+def generate_answer(question, previous_answers, model_name, open_router_key, openai_api_key,temperature,top_p):
     """Generates an answer to a question using the specified language model."""
     gen_prompt = create_gen_prompt(question, previous_answers)
     try:
         new_answer = chat_with_model(prompt=gen_prompt, model=model_name, open_router_key=open_router_key,
-                                     openai_api_key=openai_api_key)
+                                     openai_api_key=openai_api_key,temperature=temperature,top_p=top_p)
         return new_answer
     except Exception as e:
         st.error(f"Error generating answer: {str(e)}")  # Use st.error
         return None
 
 
-def evaluate_answer(question, new_answer, open_router_key, openai_api_key, judge_model_name):
+def evaluate_answer(question, new_answer, open_router_key, openai_api_key, judge_model_name,temperature,top_p):
     """Evaluates the coherence and novelty of an answer."""
     judge_prompt = create_judge_prompt(question, new_answer)
     judge = judge_model_name  # Use the judge_model_name passed to the function
     try:
         judge_response = chat_with_model(prompt=judge_prompt, model=judge, open_router_key=open_router_key,
-                                         openai_api_key=openai_api_key)
+                                         openai_api_key=openai_api_key,temperature=temperature,top_p=top_p)
         coherence_score = int(judge_response.split("<coherence_score>")[1].split("</coherence_score>")[0])
         return coherence_score
     except Exception as e:
@@ -33,18 +33,18 @@ def evaluate_answer(question, new_answer, open_router_key, openai_api_key, judge
         return None
 
 
-def process_question(question, model_name, open_router_key, openai_api_key, result_queue, judge_model_name,coherence_threshold,novelty_threshold):
+def process_question(question, model_name, open_router_key, openai_api_key, result_queue, judge_model_name,coherence_threshold,novelty_threshold,temperature,top_p):
     start_time = time.time()
     previous_answers = []
     question_novelty = 0
 
     try:
         while True:
-            new_answer = generate_answer(question, previous_answers, model_name, open_router_key, openai_api_key)
+            new_answer = generate_answer(question, previous_answers, model_name, open_router_key, openai_api_key, temperature,top_p)
             if new_answer is None:
                 break
 
-            coherence_score = evaluate_answer(question, new_answer, open_router_key, openai_api_key, judge_model_name)
+            coherence_score = evaluate_answer(question, new_answer, open_router_key, openai_api_key, judge_model_name,temperature,top_p)
             if coherence_score is None:
                 break
 
@@ -126,7 +126,7 @@ def get_novelty_score(new_answer: str, previous_answers: list, openai_api_key):
     return novelty
 
 
-def benchmark_model_multithreaded(model_name, questions, open_router_key, openai_api_key, max_threads=None, judge_model_name=None,coherence_threshold=None,novelty_threshold=None):
+def benchmark_model_multithreaded(model_name, questions, open_router_key, openai_api_key, max_threads=None, judge_model_name=None,coherence_threshold=None,novelty_threshold=None,temperature=0,top_p=0):
     novelty_score = 0
     results = []
     result_queue = queue.Queue()  # Create a queue for communication
@@ -140,7 +140,7 @@ def benchmark_model_multithreaded(model_name, questions, open_router_key, openai
     with ThreadPoolExecutor(max_workers=max_workers) as executor:
         # Submit tasks to the thread pool
         future_to_question = {
-            executor.submit(process_question, question, model_name, open_router_key, openai_api_key, result_queue, judge_model_name,coherence_threshold,novelty_threshold): question
+            executor.submit(process_question, question, model_name, open_router_key, openai_api_key, result_queue, judge_model_name,coherence_threshold,novelty_threshold,temperature,top_p): question
             for question in questions
         }
 
@@ -185,12 +185,12 @@ def benchmark_model_multithreaded(model_name, questions, open_router_key, openai
     return results
 
 
-def benchmark_model_sequential(model_name, questions, open_router_key, openai_api_key, judge_model_name,coherence_threshold,novelty_threshold):
+def benchmark_model_sequential(model_name, questions, open_router_key, openai_api_key, judge_model_name,coherence_threshold,novelty_threshold,temperature,top_p):
     novelty_score = 0
     results = []
 
     for i, question in enumerate(questions):
-        for result in process_question(question, model_name, open_router_key, openai_api_key, None, judge_model_name,coherence_threshold,novelty_threshold):
+        for result in process_question(question, model_name, open_router_key, openai_api_key, None, judge_model_name,coherence_threshold,novelty_threshold,temperature,top_p):
             if result["type"] == "answer":
                 st.write(f"**Question:** {result['question']}")
                 st.write(f"**New Answer:**\n{result['answer']}")

models.py

@@ -5,7 +5,7 @@ from retry import retry
 
 
 @retry(tries=3)
-def chat_with_model(prompt, model, open_router_key=None, openai_api_key=None, max_tokens=4000, temperature=0):
+def chat_with_model(prompt, model, open_router_key=None, openai_api_key=None, max_tokens=4000, temperature=0,top_p=0):
     if open_router_key:
         client = OpenAI(
             api_key=open_router_key,
@@ -25,7 +25,8 @@ def chat_with_model(prompt, model, open_router_key=None, openai_api_key=None, ma
             }
         ],
         max_tokens=max_tokens,
-        temperature=temperature
+        temperature=temperature,
+        top_p=top_p
     )
     return response.choices[0].message.content