x

app.py

@@ -93,6 +93,11 @@ if st.session_state.open_router_key and st.session_state.openai_api_key:
     if "user_questions" not in st.session_state:
         st.session_state.user_questions = []
 
+    # Threshold Sliders
+    st.subheader("Threshold Sliders")
+    coherence_threshold = st.slider("Coherence Threshold (0-5):", 0, 5, 3)
+    novelty_threshold = st.slider("Novelty Threshold (0-1):", 0.0, 1.0, 0.1)
+
     # Workflow Selection
     workflow = st.radio("Select Workflow:", ["Use Predefined Questions", "Use User-Defined Questions"])
 
@@ -139,6 +144,8 @@ if st.session_state.open_router_key and st.session_state.openai_api_key:
     else:
         max_threads = None  # For sequential mode
 
+
+
     # Benchmark Execution
     if st.button("Start Benchmark"):
         if not selected_questions:
@@ -152,9 +159,9 @@ if st.session_state.open_router_key and st.session_state.openai_api_key:
 
             # Benchmarking logic using the chosen execution mode
             if execution_mode == "Sequential":
-                question_results = benchmark_model_sequential(model_name, selected_questions, st.session_state.open_router_key, st.session_state.openai_api_key,judge_model_name)
+                question_results = benchmark_model_sequential(model_name, selected_questions, st.session_state.open_router_key, st.session_state.openai_api_key,judge_model_name,coherence_threshold,novelty_threshold)
             else:  # Multithreaded
-                question_results = benchmark_model_multithreaded(model_name, selected_questions, st.session_state.open_router_key, st.session_state.openai_api_key, max_threads, judge_model_name)
+                question_results = benchmark_model_multithreaded(model_name, selected_questions, st.session_state.open_router_key, st.session_state.openai_api_key, max_threads, judge_model_name, coherence_threshold,novelty_threshold)
 
             results.extend(question_results)
 

main.py

@@ -33,7 +33,7 @@ def evaluate_answer(question, new_answer, open_router_key, openai_api_key, judge
         return None
 
 
-def process_question(question, model_name, open_router_key, openai_api_key, result_queue, judge_model_name):
+def process_question(question, model_name, open_router_key, openai_api_key, result_queue, judge_model_name,coherence_threshold,novelty_threshold):
     start_time = time.time()
     previous_answers = []
     question_novelty = 0
@@ -48,12 +48,12 @@ def process_question(question, model_name, open_router_key, openai_api_key, resu
             if coherence_score is None:
                 break
 
-            if coherence_score <= 3:
+            if coherence_score <= coherence_threshold:
                 break
 
             novelty_score = get_novelty_score(new_answer, previous_answers, openai_api_key)
 
-            if novelty_score < 0.1:
+            if novelty_score < novelty_threshold:
                 break
 
 
@@ -126,7 +126,7 @@ def get_novelty_score(new_answer: str, previous_answers: list, openai_api_key):
     return novelty
 
 
-def benchmark_model_multithreaded(model_name, questions, open_router_key, openai_api_key, max_threads=None, judge_model_name=None):
+def benchmark_model_multithreaded(model_name, questions, open_router_key, openai_api_key, max_threads=None, judge_model_name=None,coherence_threshold=None,novelty_threshold=None):
     novelty_score = 0
     results = []
     result_queue = queue.Queue()  # Create a queue for communication
@@ -140,7 +140,7 @@ def benchmark_model_multithreaded(model_name, questions, open_router_key, openai
     with ThreadPoolExecutor(max_workers=max_workers) as executor:
         # Submit tasks to the thread pool
         future_to_question = {
-            executor.submit(process_question, question, model_name, open_router_key, openai_api_key, result_queue, judge_model_name): question
+            executor.submit(process_question, question, model_name, open_router_key, openai_api_key, result_queue, judge_model_name,coherence_threshold,novelty_threshold): question
             for question in questions
         }
 
@@ -185,12 +185,12 @@ def benchmark_model_multithreaded(model_name, questions, open_router_key, openai
     return results
 
 
-def benchmark_model_sequential(model_name, questions, open_router_key, openai_api_key, judge_model_name):
+def benchmark_model_sequential(model_name, questions, open_router_key, openai_api_key, judge_model_name,coherence_threshold,novelty_threshold):
     novelty_score = 0
     results = []
 
     for i, question in enumerate(questions):
-        for result in process_question(question, model_name, open_router_key, openai_api_key, None, judge_model_name):
+        for result in process_question(question, model_name, open_router_key, openai_api_key, None, judge_model_name,coherence_threshold,novelty_threshold):
             if result["type"] == "answer":
                 st.write(f"**Question:** {result['question']}")
                 st.write(f"**New Answer:**\n{result['answer']}")