Merge branch 'feat/chatbot' into 'master'

Save rough draft of chatbot

See merge request animalequality/lv-recipe-chatbot!1

.env.example

@@ -0,0 +1,2 @@
+OPENAI_API_KEY = "sk-*"
+PROMPTLAYER_API_KEY = "pl_*"

.gitignore

@@ -0,0 +1,2 @@
+.env
+*__pycache__

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "[python]": {
+        "editor.defaultFormatter": "ms-python.black-formatter"
+    },
+    "python.formatting.provider": "none"
+}

README.md

@@ -0,0 +1,43 @@
+# Vegan Recipe Chatbot
+
+![Screenshot of Chatbot initial interface](docs/assets/chatbot_init.png)
+
+## Quickstart
+
+`git clone` the repo  
+
+```sh
+cd lv-recipe-chatbot
+```  
+
+Install Python poetry for dependency management.  
+
+```sh
+poetry install
+```
+
+Put API secrets in .env
+
+```sh
+cp .env.example .env
+# edit .env with your secret key(s). Only OPEN_AI_KEY is required.
+```
+
+One option is to enter the poetry environment.  
+
+```sh
+poetry shell
+```
+
+Then start the Gradio demo.
+
+```sh
+python app.py
+```
+
+## Useful links
+
+* [Task Matrix (Formerly Visual ChatGPT)](https://github.com/microsoft/TaskMatrix)
+* [LangChain](https://python.langchain.com/en/latest/index.html)
+* [LLM Prompt Engineering](https://www.promptingguide.ai)
+* [OpenAI best practices for prompts](https://help.openai.com/en/articles/6654000-best-practices-for-prompt-engineering-with-openai-api)

chatbot/__init__.py

@@ -0,0 +1,3 @@
+from dotenv import load_dotenv
+
+load_dotenv()

chatbot/app.py

@@ -0,0 +1,172 @@
+import gradio as gr
+from langchain.chat_models import ChatOpenAI
+from langchain.chains import ConversationChain
+from langchain.memory import ConversationBufferMemory
+
+from langchain.prompts.chat import (
+    HumanMessagePromptTemplate,
+    MessagesPlaceholder,
+    ChatPromptTemplate,
+)
+from chatbot.engineer_prompt import init_prompt
+
+# from transformers import (
+#     BlipProcessor,
+#     BlipForConditionalGeneration,
+#     BlipForQuestionAnswering,
+# )
+# import torch
+# from PIL import Image
+
+# class ImageCaptioning:
+#     def __init__(self, device):
+#         print(f"Initializing ImageCaptioning to {device}")
+#         self.device = device
+#         self.torch_dtype = torch.float16 if "cuda" in device else torch.float32
+#         self.processor = BlipProcessor.from_pretrained(
+#             "Salesforce/blip-image-captioning-base"
+#         )
+#         self.model = BlipForConditionalGeneration.from_pretrained(
+#             "Salesforce/blip-image-captioning-base", torch_dtype=self.torch_dtype
+#         ).to(self.device)
+
+#     def inference(self, image_path):
+#         inputs = self.processor(Image.open(image_path), return_tensors="pt").to(
+#             self.device, self.torch_dtype
+#         )
+#         out = self.model.generate(**inputs)
+#         captions = self.processor.decode(out[0], skip_special_tokens=True)
+#         print(
+#             f"\nProcessed ImageCaptioning, Input Image: {image_path}, Output Text: {captions}"
+#         )
+#         return captions
+
+
+# class VisualQuestionAnswering:
+#     def __init__(self, device):
+#         print(f"Initializing VisualQuestionAnswering to {device}")
+#         self.torch_dtype = torch.float16 if "cuda" in device else torch.float32
+#         self.device = device
+#         self.processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base")
+#         self.model = BlipForQuestionAnswering.from_pretrained(
+#             "Salesforce/blip-vqa-base", torch_dtype=self.torch_dtype
+#         ).to(self.device)
+
+#     def inference(self, image_path, question):
+#         raw_image = Image.open(image_path).convert("RGB")
+#         inputs = self.processor(raw_image, question, return_tensors="pt").to(
+#             self.device, self.torch_dtype
+#         )
+#         out = self.model.generate(**inputs)
+#         answer = self.processor.decode(out[0], skip_special_tokens=True)
+#         print(
+#             f"\nProcessed VisualQuestionAnswering, Input Image: {image_path}, Input Question: {question}, "
+#             f"Output Answer: {answer}"
+#         )
+#         return
+
+
+class ConversationBot:
+    def __init__(
+        self,
+    ):
+        self.chat = ChatOpenAI(temperature=1, verbose=True)
+        self.memory = ConversationBufferMemory(return_messages=True)
+        self.init_prompt_msgs = init_prompt.messages
+        self.ai_prompt_questions = {
+            "ingredients": self.init_prompt_msgs[1],
+            "allergies": self.init_prompt_msgs[3],
+            "recipe_open_params": self.init_prompt_msgs[5],
+        }
+
+    def respond(self, user_msg, chat_history):
+        response = self._get_bot_response(user_msg, chat_history)
+        chat_history.append((user_msg, response))
+        return "", chat_history
+
+    def init_conversation(self, formatted_chat_prompt):
+        self.conversation = ConversationChain(
+            llm=self.chat,
+            memory=self.memory,
+            prompt=formatted_chat_prompt,
+            verbose=True,
+        )
+
+    def reset(self):
+        self.memory.clear()
+
+    def _get_bot_response(self, user_msg: str, chat_history) -> str:
+        if len(chat_history) < 2:
+            return self.ai_prompt_questions["allergies"].prompt.template
+
+        if len(chat_history) < 3:
+            return self.ai_prompt_questions["recipe_open_params"].prompt.template
+
+        if len(chat_history) < 4:
+            user = 0
+            ai = 1
+            user_msgs = [msg_pair[user] for msg_pair in chat_history[1:]]
+            f_init_prompt = init_prompt.format_prompt(
+                ingredients=user_msgs[0],
+                allergies=user_msgs[1],
+                recipe_freeform_input=user_msg,
+            )
+            chat_msgs = f_init_prompt.to_messages()
+            results = self.chat.generate([chat_msgs])
+            chat_msgs.extend(
+                [
+                    results.generations[0][0].message,
+                    MessagesPlaceholder(variable_name="history"),
+                    HumanMessagePromptTemplate.from_template("{input}"),
+                ]
+            )
+            open_prompt = ChatPromptTemplate.from_messages(chat_msgs)
+            # prepare the open conversation chain from this point
+            self.init_conversation(open_prompt)
+            return results.generations[0][0].message.content
+
+        response = self.conversation.predict(input=user_msg)
+        return response
+
+    # def run_image(self, image, state, txt, lang):
+    #     image_filename = os.path.join("image", f"{str(uuid.uuid4())[:8]}.png")
+    #     print("======>Auto Resize Image...")
+    #     img = Image.open(image.name)
+    #     width, height = img.size
+    #     ratio = min(512 / width, 512 / height)
+    #     width_new, height_new = (round(width * ratio), round(height * ratio))
+    #     width_new = int(np.round(width_new / 64.0)) * 64
+    #     height_new = int(np.round(height_new / 64.0)) * 64
+    #     img = img.resize((width_new, height_new))
+    #     img = img.convert("RGB")
+    #     img.save(image_filename, "PNG")
+    #     print(f"Resize image form {width}x{height} to {width_new}x{height_new}")
+    #     description = self.models["ImageCaptioning"].inference(image_filename)
+    #     Human_prompt = f'\nHuman: provide a figure named {image_filename}. The description is: {description}. This information helps you to understand this image, but you should use tools to finish following tasks, rather than directly imagine from my description. If you understand, say "Received". \n'
+    #     self.memory.buffer = (
+    #         self.agent.memory.buffer + Human_prompt + "AI: " + AI_prompt
+    #     )
+    #     state = state + [(f"![](file={image_filename})*{image_filename}*", AI_prompt)]
+    #     print(
+    #         f"\nProcessed run_image, Input image: {image_filename}\nCurrent state: {state}\n"
+    #         f"Current Memory: {self.agent.memory.buffer}"
+    #     )
+    #     return state, state, f"{txt} {image_filename} "
+
+
+with gr.Blocks() as demo:
+    bot = ConversationBot()
+    chatbot = gr.Chatbot(
+        value=[(None, bot.ai_prompt_questions["ingredients"].prompt.template)]
+    )
+
+    msg = gr.Textbox()
+    clear = gr.Button("Clear")
+
+    msg.submit(
+        fn=bot.respond, inputs=[msg, chatbot], outputs=[msg, chatbot], queue=False
+    )
+    clear.click(lambda: None, None, chatbot, queue=False).then(bot.reset)
+
+if __name__ == "__main__":
+    demo.launch()

chatbot/engineer_prompt.py

@@ -0,0 +1,105 @@
+from langchain.chat_models import PromptLayerChatOpenAI
+from langchain.schema import HumanMessage, AIMessage, SystemMessage
+from langchain.chains import ConversationChain
+from langchain.memory import ConversationBufferMemory
+from langchain.prompts.chat import (
+    ChatPromptTemplate,
+    SystemMessagePromptTemplate,
+    HumanMessagePromptTemplate,
+    AIMessagePromptTemplate,
+    MessagesPlaceholder,
+)
+
+# TODO Multiple chains sequenced?
+# I think your way works fine, though you'd probably want to wrap it up in some initializer so you can "initialize" the chain via LLM calls. I'd probably use 2 chains and have a wrapping chain switch from the first to the second after initializing.
+# https://discord.com/channels/1038097195422978059/1038097349660135474/1100533951136800828
+
+init_prompt = ChatPromptTemplate.from_messages(
+    [
+        SystemMessagePromptTemplate.from_template(
+            """
+The following is a conversation between a human and a friendly AI chef. 
+The AI is compassionate to animals and only recommends vegan recipes based on the ingredients, allergies, and other preferences the human has.
+
+Knowledge: A vegan diet implies a plant-based diet avoiding all animal foods such as meat (including fish, shellfish and insects), dairy, eggs and honey 
+
+Let's think step by step.
+If the human messages are unrelated to vegan recipes, remind them of your purpose to recommend vegan recipes. 
+    """.strip()
+        ),
+        AIMessagePromptTemplate.from_template(
+            "What ingredients do you wish to cook with?"
+        ),
+        HumanMessagePromptTemplate.from_template("Ingredients: {ingredients}"),
+        AIMessagePromptTemplate.from_template(
+            "Do you have any allergies I should be aware of?"
+        ),
+        HumanMessagePromptTemplate.from_template("Allergies: {allergies}"),
+        AIMessagePromptTemplate.from_template(
+            "Do you have any preferences I should consider for the recipe such as preparation time, difficulty, or cuisine region?"
+        ),
+        HumanMessagePromptTemplate.from_template(
+            """
+Give me a vegan recipe that includes at least a few of the ingredients provided (if any).
+Respect the human's allergies (if any).
+Follow these other preferences as closely as possible if they are inline with your purpose of recommending vegan recipes:
+
+###
+Preferences: {recipe_freeform_input}
+###
+
+Output format:
+
+**Vegan recipe name**
+Preparation time (humanized)
+
+Ingredients (List of ingredients with quantities):
+- <quantity and unit> <ingredient>
+
+Steps (detailed):
+1.
+2.
+3.
+...
+""".strip()
+        ),
+    ]
+)
+
+
+if __name__ == "__main__":
+    chat = PromptLayerChatOpenAI(
+        temperature=1, pl_tags=["langchain"], return_pl_id=True
+    )
+    memory = ConversationBufferMemory(return_messages=True)
+    chat_msgs = init_prompt.format_prompt(
+        ingredients="tofu, pickles, olives, tomatoes, lettuce, bell peppers, carrots, bread",
+        allergies="",
+        recipe_freeform_input="The preparation time should be less than 30 minutes. I really love Thai food!",
+    )
+
+    chat_msgs = chat_msgs.to_messages()
+    results = chat.generate([chat_msgs])
+    chat_msgs.extend(
+        [
+            results.generations[0][0].message,
+            MessagesPlaceholder(variable_name="history"),
+            HumanMessagePromptTemplate.from_template("{input}"),
+        ]
+    )
+    open_prompt = ChatPromptTemplate.from_messages(chat_msgs)
+    conversation = ConversationChain(
+        llm=chat, verbose=True, memory=memory, prompt=open_prompt
+    )
+
+    result = conversation.predict(input="Recommend a different recipe please.")
+    print(result)
+
+#! PL score example
+# chat_results = chat.generate([[HumanMessage(content=prompt)]])
+
+# for res in chat_results.generations:
+#     pl_request_id = res[0].generation_info["pl_request_id"]
+#     print(res[0].text)
+#     score = int(input("Enter a score from 0 to 100 for how the prompt performed: "))
+#     promptlayer.track.score(request_id=pl_request_id, score=score)

pyproject.toml

@@ -0,0 +1,31 @@
+[tool.poetry]
+name = "lv-recipe-chatbot"
+version = "0.1.0"
+description = "Chatbot for recommending vegan recipes"
+authors = ["Evan Lesmez <[email protected]>"]
+readme = "README.md"
+packages = [{ include = "chatbot" }]
+
+[tool.poetry.dependencies]
+python = "^3.8.1"
+langchain = "^0.0.145"
+openai = "^0.27.4"
+gradio = "^3.27.0"
+jupyterlab = "^3.6.3"
+tqdm = "^4.65.0"
+transformers = "^4.28.1"
+promptlayer = "^0.1.80"
+python-dotenv = "^1.0.0"
+torch = "1.13.1"
+torchvision = "0.14.1"
+wget = "3.2"
+
+[tool.poetry.group.dev.dependencies]
+black = "^23.3.0"
+pytest = "^7.3.1"
+mypy = "^1.2.0"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"