Auto QA

app.py

@@ -8,8 +8,9 @@ import json
 from langchain.callbacks import get_openai_callback
 from langchain_openai import ChatOpenAI
 import base64
-from chains import rag_chain, parse_model_response
+from chat_chains import rag_chain, parse_model_response
 from langchain_core.messages import AIMessage, HumanMessage
+from autoqa_chains import auto_qa_chain, followup_qa_chain, auto_qa_output_parser
 
 st.set_page_config(layout="wide")
 os.environ["OPENAI_API_KEY"] = "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"
@@ -41,10 +42,11 @@ Here's a quick guide to getting started with me:
 
 | Command | Description |
 |---------|-------------|
-| `/upload` | Upload and process documents for our conversation. |
+| `/upload` <list of urls> | Upload and process documents for our conversation. |
 | `/index` | View an index of processed documents to easily navigate your research. |
 | `/cost` | Calculate the cost of our conversation, ensuring transparency in resource usage. |
 | `/download` | Download conversation data for your records or further analysis. |
+| `/auto` <document id> | Automatically generate questions and answers for a document. |
 
 <br>
 
@@ -55,7 +57,7 @@ Use `/man` at any point of time to view this guide again.
 
 
 def process_documents_wrapper(inputs):
-    snippets = process_documents(inputs)
+    snippets, documents = process_documents(inputs)
     st.session_state.retriever = create_retriever(snippets)
     st.session_state.source_doc_urls = inputs
     st.session_state.index = [
@@ -63,6 +65,7 @@ def process_documents_wrapper(inputs):
     ]
     response = f"Uploaded and processed documents {inputs}"
     st.session_state.messages.append((f"/upload {inputs}", response, ""))
+    st.session_state.documents = documents
     return response
 
 
@@ -163,6 +166,38 @@ def query_llm_wrapper(inputs):
     return answer, citations
 
 
+def auto_qa_chain_wrapper(inputs):
+    document = st.session_state.documents[inputs]
+    llm = ChatOpenAI(model="gpt-4-turbo-preview", temperature=0)
+    auto_qa_conversation = []
+    with get_openai_callback() as cb:
+        auto_qa_response = auto_qa_chain(llm).invoke({"paper": document})
+        auto_qa_response_parsed = auto_qa_output_parser.invoke(auto_qa_response)[
+            "questions"
+        ]
+        auto_qa_conversation = [
+            (f'/auto {qa["question"]}', qa["answer"], "")
+            for qa in auto_qa_response_parsed
+        ]
+        stats = cb
+    st.session_state.messages.append(
+        (f"/auto {inputs}", "Auto Convervation Generated", "")
+    )
+    for qa in auto_qa_conversation:
+        st.session_state.messages.append((qa[0], qa[1], ""))
+
+    st.session_state.costing.append(
+        {
+            "prompt tokens": stats.prompt_tokens,
+            "completion tokens": stats.completion_tokens,
+            "cost": stats.total_cost,
+        }
+    )
+    return "\n\n".join(
+        f"Q: {qa['question']}\n\nA: {qa['answer']}" for qa in auto_qa_response_parsed
+    )
+
+
 def boot(command_center):
     st.write("# Agent Zeta")
     if "costing" not in st.session_state:
@@ -196,6 +231,7 @@ if __name__ == "__main__":
         ("/cost", None, calculate_cost_wrapper),
         ("/download", None, download_conversation_wrapper),
         ("/man", None, lambda x: welcome_message),
+        ("/auto", int, auto_qa_chain_wrapper),
     ]
     command_center = CommandCenter(
         default_input_type=str,

autoqa_chains.py

@@ -0,0 +1,54 @@
+from langchain_core.pydantic_v1 import BaseModel, Field
+from typing import List
+from langchain_core.output_parsers import JsonOutputParser
+from langchain_core.prompts import PromptTemplate
+
+
+class QA(BaseModel):
+    question: str = Field(description="question")
+    answer: str = Field(description="answer")
+
+
+class AutoQA(BaseModel):
+    questions: List[QA] = Field(description="list of question and answers")
+
+
+qa_prompt_template = """
+Come up with the 10 questions and answers that could be commonly asked by people about the following research paper.
+The question and answers should capture the whole essence of the research paper
+The answers should be a bit detailed and strictly based on the research paper.
+Your response should be recorded in the following json format: {format_instructions}.
+
+here is the research paper: ####{paper}####
+"""
+
+auto_qa_output_parser = JsonOutputParser(pydantic_object=AutoQA)
+qa_prompt = PromptTemplate(
+    template=qa_prompt_template,
+    input_variables=["paper"],
+    partial_variables={
+        "format_instructions": auto_qa_output_parser.get_format_instructions()
+    },
+)
+auto_qa_chain = lambda model: qa_prompt | model
+
+
+followup_prompt_template = """
+Question: {question}
+Answer: {answer}
+Based on the above question and answer and the research paper as your context, come up with a followup question and its answer.
+The answer should be a bit detailed and strictly based on the research paper.
+Your response should be recorded in the following json format: {format_instructions}.
+
+here is the research paper: ####{paper}####
+"""
+
+followup_prompt = PromptTemplate(
+    template=followup_prompt_template,
+    input_variables=["paper"],
+    partial_variables={
+        "format_instructions": auto_qa_output_parser.get_format_instructions()
+    },
+)
+
+followup_qa_chain = lambda model: qa_prompt | model

chat_chains.py

@@ -0,0 +1,107 @@
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.runnables import RunnablePassthrough
+import xml.etree.ElementTree as ET
+import re
+
+contextualize_q_system_prompt = """Given a chat history and the latest user question \
+which might reference context in the chat history, formulate a standalone question \
+which can be understood without the chat history. Do NOT answer the question, \
+just reformulate it if needed and otherwise return it as is."""
+contextualize_q_prompt = ChatPromptTemplate.from_messages(
+    [
+        ("system", contextualize_q_system_prompt),
+        MessagesPlaceholder(variable_name="chat_history"),
+        ("human", "{question}"),
+    ]
+)
+contextualize_q_chain = lambda llm: contextualize_q_prompt | llm | StrOutputParser()
+
+qa_system_prompt = """As Zeta, your mission is to assist users in navigating the vast sea of machine learning research with ease and insight. When responding to inquiries, adhere to the following guidelines to ensure the utmost accuracy and utility:
+
+Contextual Understanding: When presented with a question, apply your understanding of machine learning concepts to interpret the context provided accurately. Utilize this context to guide your search for answers within the specified research papers.
+
+Answer Provision: Always provide an answer that is directly supported by the research papers' content. If the information needed to answer the question is not available, clearly state, "I don't know."
+
+Citation Requirement: For every answer given, include multiple citations from the research papers. A citation must include a direct quote from the paper that supports your answer, along with the identification (ID) of the paper. This ensures that all provided information can be traced back to its source, maintaining a high level of credibility and transparency.
+
+Formatting Guidelines: Present your citations in the following structured format at the end of your answer to maintain clarity and consistency:
+
+
+<citations>
+    <citation><source_id>[Source ID]</source_id><quote>[Direct quote from the source]</quote></citation>
+    ...
+</citations>
+
+
+Conflict Resolution: In cases where multiple sources offer conflicting information, evaluate the context, relevance, and credibility of each source to determine the most accurate answer. Explain your reasoning within the citation section to provide insight into your decision-making process.
+
+User Engagement: Encourage user engagement by asking clarifying questions if the initial inquiry is ambiguous or lacks specific context. This helps in providing more targeted and relevant responses.
+
+Continual Learning: Although you are not expected to generate new text or insights beyond the provided papers, be open to learning from new information as it becomes available to you through user interactions and queries.
+
+By following these guidelines, you ensure that users receive valuable, accurate, and source-backed insights into their inquiries, making their exploration of machine learning research more productive and enlightening.
+
+{context}"""
+qa_prompt = ChatPromptTemplate.from_messages(
+    [
+        ("system", qa_system_prompt),
+        MessagesPlaceholder(variable_name="chat_history"),
+        ("human", "{question}"),
+    ]
+)
+
+
+def format_docs(docs):
+    return "\n\n".join(
+        f"{doc.metadata['chunk_id']}: {doc.page_content}" for doc in docs
+    )
+
+
+def contextualized_question(input: dict):
+    if input.get("chat_history"):
+        return contextualize_q_chain
+    else:
+        return input["question"]
+
+
+rag_chain = lambda retriever, llm: (
+    RunnablePassthrough.assign(
+        context=contextualized_question | retriever | format_docs
+    )
+    | qa_prompt
+    | llm
+)
+
+
+def parse_model_response(input_string):
+    parsed_data = {"answer": "", "citations": []}
+    xml_matches = re.findall(r"<citations>.*?</citations>", input_string, re.DOTALL)
+    if not xml_matches:
+        parsed_data["answer"] = input_string
+        return parsed_data
+
+    outside_text_parts = []
+    last_end_pos = 0
+
+    for xml_string in xml_matches:
+        match = re.search(re.escape(xml_string), input_string[last_end_pos:], re.DOTALL)
+
+        if match:
+            outside_text_parts.append(
+                input_string[last_end_pos : match.start() + last_end_pos]
+            )
+            last_end_pos += match.end()
+
+        root = ET.fromstring(xml_string)
+
+        for citation in root.findall("citation"):
+            source_id = citation.find("source_id").text
+            quote = citation.find("quote").text
+            parsed_data["citations"].append({"source_id": source_id, "quote": quote})
+
+    outside_text_parts.append(input_string[last_end_pos:])
+
+    parsed_data["answer"] = "".join(outside_text_parts)
+
+    return parsed_data