deploy 5

api/__main__.py

@@ -26,7 +26,7 @@ def get_answer(question: str, messgages_context: str):
         f"Received request with question: {question}" \
         f"and context: {messgages_context}"
     )
-    response = qa_model.get_answer(
+    response = qa_model.get_response(
         question=question,
         messages_context=messgages_context
     )

api/question_answering/qa_model.py

@@ -17,13 +17,15 @@ from langchain.vectorstores import FAISS
 from api.logger import logger
 from api.question_answering.response import Response
 
-"""
+
 class LocalBinaryModel(LLM):
     model_id: str = None
-    llm: Llama = None
+    llm: None = None
 
     def __init__(self, model_id: str = None):
         super().__init__()
+        from llama_cpp import Llama
+
         model_path = f'api/question_answering/{model_id}'
         if not os.path.exists(model_path):
             raise ValueError(f'{model_path} does not exist')
@@ -48,7 +50,7 @@ class LocalBinaryModel(LLM):
     @property
     def _llm_type(self) -> str:
         return self.model_id
-"""
+
 
 class TransformersPipelineModel(LLM):
     model_id: str = None
@@ -169,9 +171,9 @@ class QAModel():
 
         if 'local_models/' in llm_model_id:
             logger.info('using local binary model')
-            #self.llm_model = LocalBinaryModel(
-            #    model_id=llm_model_id
-            #)
+            self.llm_model = LocalBinaryModel(
+                model_id=llm_model_id
+            )
         elif 'api_models/' in llm_model_id:
             logger.info('using api served model')
             self.llm_model = APIServedModel(
@@ -216,7 +218,7 @@ class QAModel():
             self.knowledge_index = FAISS.load_local(f"./indexes/run/", embedding_model)
 
 
-    def get_answer(self, question: str, messages_context: str = '') -> Response:
+    def get_response(self, question: str, messages_context: str = '') -> Response:
         """
         Generate an answer to the specified question.
 

app.py

@@ -23,11 +23,11 @@ with gr.Blocks() as demo:
     def respond(message, chat_history):
         context = "".join(f"User: {msg} \nBot:{bot_msg}\n" for msg, bot_msg in chat_history)
         logger.info(f"Context: {context}")
-        answer = model.get_answer(message, context)
-        bot_message = answer.get_response() + answer.get_sources_as_text() + "\n"
+        response = model.get_response(message, context)
+        bot_message = response.get_answer() + response.get_sources_as_text() + "\n"
         chat_history.append((message, bot_message))
         return "", chat_history
 
     msg.submit(respond, [msg, chatbot], [msg, chatbot])
 
-demo.launch(share=False, debug=True)
+demo.launch(share=True, debug=True)

requirements.txt

@@ -2,11 +2,13 @@ transformers
 accelerate
 einops
 huggingface_hub
+gradio
 beautifulsoup4==4.12.0
 discord.py==2.2.2
 evaluate==0.4.0
 fastapi==0.98.0
 langchain==0.0.154
+llama_cpp_python==0.1.39
 nltk==3.8.1
 nbconvert==7.6.0
 nbformat==5.9.0