Upload 3 files

app.py

@@ -7,54 +7,90 @@ from llmdolphin import (get_llm_formats, get_dolphin_model_format,
     get_dolphin_models, get_dolphin_model_info, select_dolphin_model,
     select_dolphin_format, add_dolphin_models, get_dolphin_sysprompt,
     get_dolphin_sysprompt_mode, select_dolphin_sysprompt, get_dolphin_languages,
-    select_dolphin_language, dolphin_respond, dolphin_parse)
+    select_dolphin_language, dolphin_respond, dolphin_parse, respond_playground)
 
+css = """
+.title { text-align: center; }
+"""
 
-with gr.Blocks(theme='NoCrypt/miku@>=1.2.2', fill_width=True, css="", delete_cache=(60, 3600)) as app:
-    gr.Markdown("""# Natural Text to SD Prompt Translator With LLM alpha
-                    Text in natural language (English, Japanese, ...) => Prompt
-                """)
-    with gr.Column():
-        with gr.Group():
-            chatbot = gr.Chatbot(likeable=False, show_copy_button=True, show_share_button=False, layout="bubble", container=True)
-            with gr.Row():
-                chat_msg = gr.Textbox(show_label=False, placeholder="Input text in English, Japanese, or any other languages and press Enter or click Send.", scale=4)
-                chat_submit = gr.Button("Send", scale=1)
-                chat_clear = gr.Button("Clear", scale=1)
-            with gr.Accordion("Additional inputs", open=False):
-                chat_format = gr.Dropdown(choices=get_llm_formats(), value=get_dolphin_model_format(get_dolphin_models()[0][1]), label="Message format")
-                chat_sysmsg = gr.Textbox(value=get_dolphin_sysprompt(), label="System message")
-                with gr.Row():
-                    chat_tokens = gr.Slider(minimum=1, maximum=4096, value=512, step=1, label="Max tokens")
-                    chat_temperature = gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature")
-                    chat_topp = gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p")
-                    chat_topk = gr.Slider(minimum=0, maximum=100, value=40, step=1, label="Top-k")
-                    chat_rp = gr.Slider(minimum=0.0, maximum=2.0, value=1.1, step=0.1, label="Repetition penalty")
-            with gr.Accordion("Add models", open=False):
-                chat_add_text = gr.Textbox(label="URL or Repo ID", placeholder="https://huggingface.co/mradermacher/MagnumChronos-i1-GGUF/blob/main/MagnumChronos.i1-Q4_K_M.gguf", lines=1)
-                chat_add_format = gr.Dropdown(choices=get_llm_formats(), value=get_llm_formats()[0], label="Message format")
-                chat_add_submit = gr.Button("Update lists of models")
-            with gr.Accordion("Modes", open=True):
-                chat_model = gr.Dropdown(choices=get_dolphin_models(), value=get_dolphin_models()[0][1], allow_custom_value=True, label="Model")
-                chat_model_info = gr.Markdown(value=get_dolphin_model_info(get_dolphin_models()[0][1]), label="Model info")
-                with gr.Row():
-                    chat_mode = gr.Dropdown(choices=get_dolphin_sysprompt_mode(), value=get_dolphin_sysprompt_mode()[0], allow_custom_value=False, label="Mode")
-                    chat_lang = gr.Dropdown(choices=get_dolphin_languages(), value="English", allow_custom_value=True, label="Output language")
-        with gr.Row():
-            with gr.Group():
-                output_text = gr.TextArea(label="Output tags", interactive=False, show_copy_button=True)
-                copy_btn = gr.Button(value="Copy to clipboard", size="sm", interactive=False)
+with gr.Blocks(theme='NoCrypt/miku@>=1.2.2', fill_width=True, css=css, delete_cache=(60, 3600)) as app:
+    with gr.Tab("Prompt Translator"):
+        with gr.Column():
+            gr.Markdown("""# Natural Text to SD Prompt Translator With LLM alpha
+                            Text in natural language (English, Japanese, ...) => Prompt
+                        """, elem_classes="title")
             with gr.Group():
-                output_text_pony = gr.TextArea(label="Output tags (Pony e621 style)", interactive=False, show_copy_button=True)
-                copy_btn_pony = gr.Button(value="Copy to clipboard", size="sm", interactive=False)
-            with gr.Accordion(label="Advanced options", open=False, visible=False):
-                tag_type = gr.Radio(label="Output tag conversion", info="danbooru for Animagine, e621 for Pony.", choices=["danbooru", "e621"], value="e621", visible=False)
-                dummy_np = gr.Textbox(label="Negative prompt", value="", visible=False)
-                dummy_np_pony = gr.Textbox(label="Negative prompt", value="", visible=False)
-                recom_animagine = gr.Textbox(label="Animagine reccomended prompt", value="Animagine", visible=False)
-                recom_pony = gr.Textbox(label="Pony reccomended prompt", value="Pony", visible=False)
-        generate_image_btn = gr.Button(value="GENERATE IMAGE", size="lg", variant="primary")
-        result_image = gr.Gallery(label="Generated images", columns=1, object_fit="contain", container=True, preview=True, show_label=False, show_share_button=False, show_download_button=True, interactive=False, visible=True, format="png")
+                chatbot = gr.Chatbot(likeable=False, show_copy_button=True, show_share_button=False, layout="bubble", container=True)
+                with gr.Row():
+                    chat_msg = gr.Textbox(show_label=False, placeholder="Input text in English, Japanese, or any other languages and press Enter or click Send.", lines=2, scale=4)
+                    chat_submit = gr.Button("Send", scale=1, variant="primary")
+                    chat_clear = gr.Button("Clear", scale=1, variant="secondary")
+                with gr.Accordion("Additional inputs", open=False):
+                    chat_format = gr.Dropdown(choices=get_llm_formats(), value=get_dolphin_model_format(get_dolphin_models()[0][1]), label="Message format")
+                    chat_sysmsg = gr.Textbox(value=get_dolphin_sysprompt(), label="System message")
+                    with gr.Row():
+                        chat_tokens = gr.Slider(minimum=1, maximum=4096, value=512, step=1, label="Max tokens")
+                        chat_temperature = gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature")
+                        chat_topp = gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p")
+                        chat_topk = gr.Slider(minimum=0, maximum=100, value=40, step=1, label="Top-k")
+                        chat_rp = gr.Slider(minimum=0.0, maximum=2.0, value=1.1, step=0.1, label="Repetition penalty")
+                with gr.Accordion("Add models", open=False):
+                    chat_add_text = gr.Textbox(label="URL or Repo ID", placeholder="https://huggingface.co/mradermacher/MagnumChronos-i1-GGUF/blob/main/MagnumChronos.i1-Q4_K_M.gguf", lines=1)
+                    chat_add_format = gr.Dropdown(choices=get_llm_formats(), value=get_llm_formats()[0], label="Message format")
+                    chat_add_submit = gr.Button("Update lists of models")
+                with gr.Accordion("Modes", open=True):
+                    chat_model = gr.Dropdown(choices=get_dolphin_models(), value=get_dolphin_models()[0][1], allow_custom_value=True, label="Model")
+                    chat_model_info = gr.Markdown(value=get_dolphin_model_info(get_dolphin_models()[0][1]), label="Model info")
+                    with gr.Row():
+                        chat_mode = gr.Dropdown(choices=get_dolphin_sysprompt_mode(), value=get_dolphin_sysprompt_mode()[0], allow_custom_value=False, label="Mode")
+                        chat_lang = gr.Dropdown(choices=get_dolphin_languages(), value="English", allow_custom_value=True, label="Output language")
+            with gr.Row():
+                with gr.Group():
+                    output_text = gr.TextArea(label="Output tags", interactive=False, show_copy_button=True)
+                    copy_btn = gr.Button(value="Copy to clipboard", size="sm", interactive=False)
+                with gr.Group():
+                    output_text_pony = gr.TextArea(label="Output tags (Pony e621 style)", interactive=False, show_copy_button=True)
+                    copy_btn_pony = gr.Button(value="Copy to clipboard", size="sm", interactive=False)
+                with gr.Accordion(label="Advanced options", open=False, visible=False):
+                    tag_type = gr.Radio(label="Output tag conversion", info="danbooru for Animagine, e621 for Pony.", choices=["danbooru", "e621"], value="e621", visible=False)
+                    dummy_np = gr.Textbox(label="Negative prompt", value="", visible=False)
+                    dummy_np_pony = gr.Textbox(label="Negative prompt", value="", visible=False)
+                    recom_animagine = gr.Textbox(label="Animagine reccomended prompt", value="Animagine", visible=False)
+                    recom_pony = gr.Textbox(label="Pony reccomended prompt", value="Pony", visible=False)
+            generate_image_btn = gr.Button(value="GENERATE IMAGE", size="lg", variant="primary")
+            with gr.Row():
+                result_image = gr.Gallery(label="Generated images", columns=1, object_fit="contain", container=True, preview=True, show_label=False, show_share_button=False, show_download_button=True, interactive=False, visible=True, format="png")
+    with gr.Tab("GGUF-Playground"):
+        gr.Markdown("""# Chat with lots of Models and LLMs using llama.cpp
+                    This tab is copy of [CaioXapelaum/GGUF-Playground](https://huggingface.co/spaces/CaioXapelaum/GGUF-Playground).<br>
+                    Don't worry about the strange appearance, **it's just a bug of Gradio!**""", elem_classes="title")
+        pg_chatbot = gr.Chatbot(scale=1, likeable=False, show_copy_button=True, show_share_button=False)
+        with gr.Accordion("Additional inputs", open=False):
+            pg_chat_model = gr.Dropdown(choices=get_dolphin_models(), value=get_dolphin_models()[0][1], allow_custom_value=True, label="Model")
+            pg_chat_model_info = gr.Markdown(value=get_dolphin_model_info(get_dolphin_models()[0][1]), label="Model info")
+            pg_chat_format = gr.Dropdown(choices=get_llm_formats(), value=get_dolphin_model_format(get_dolphin_models()[0][1]), label="Message format")
+            pg_chat_sysmsg = gr.Textbox(value="You are a helpful assistant.", label="System message")
+            with gr.Row():
+                pg_chat_tokens = gr.Slider(minimum=1, maximum=4096, value=2048, step=1, label="Max tokens")
+                pg_chat_temperature = gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature")
+                pg_chat_topp = gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p")
+                pg_chat_topk = gr.Slider(minimum=0, maximum=100, value=40, step=1, label="Top-k")
+                pg_chat_rp = gr.Slider(minimum=0.0, maximum=2.0, value=1.1, step=0.1, label="Repetition penalty")
+            with gr.Accordion("Add models", open=True):
+                pg_chat_add_text = gr.Textbox(label="URL or Repo ID", placeholder="https://huggingface.co/mradermacher/MagnumChronos-i1-GGUF/blob/main/MagnumChronos.i1-Q4_K_M.gguf", lines=1)
+                pg_chat_add_format = gr.Dropdown(choices=get_llm_formats(), value=get_llm_formats()[0], label="Message format")
+                pg_chat_add_submit = gr.Button("Update lists of models")
+        gr.ChatInterface(
+            fn=respond_playground,
+            #title="Chat with lots of Models and LLMs using llama.cpp",
+            retry_btn="Retry",
+            undo_btn="Undo",
+            clear_btn="Clear",
+            submit_btn="Send",
+            #additional_inputs_accordion='gr.Accordion(label="Additional Inputs", open=False)',
+            additional_inputs=[pg_chat_model, pg_chat_sysmsg, pg_chat_tokens, pg_chat_temperature, pg_chat_topp, pg_chat_topk, pg_chat_rp],
+            chatbot=pg_chatbot
+        )
     gr.LoginButton()
     gr.DuplicateButton(value="Duplicate Space for private use (This demo does not work on CPU. Requires GPU Space)")
 
@@ -91,6 +127,18 @@ with gr.Blocks(theme='NoCrypt/miku@>=1.2.2', fill_width=True, css="", delete_cac
 
     generate_image_btn.click(generate_image, [output_text, dummy_np], [result_image], show_progress="full")
 
+    pg_chat_model.change(select_dolphin_model, [pg_chat_model], [pg_chat_model, pg_chat_format, pg_chat_model_info], queue=True, show_progress="full")#\
+    #.success(lambda: None, None, pg_chatbot, queue=False)
+    pg_chat_format.change(select_dolphin_format, [pg_chat_format], [pg_chat_format], queue=False)#\
+    #.success(lambda: None, None, pg_chatbot, queue=False)
+    gr.on(
+        triggers=[pg_chat_add_text.submit, pg_chat_add_submit.click],
+        fn=add_dolphin_models,
+        inputs=[pg_chat_add_text, pg_chat_add_format],
+        outputs=[pg_chat_model],
+        queue=False,
+        trigger_mode="once",
+    )
 
 if __name__ == "__main__":
     app.queue()

llmdolphin.py

@@ -7,6 +7,7 @@ from llama_cpp_agent.chat_history import BasicChatHistory
 from llama_cpp_agent.chat_history.messages import Roles
 from ja_to_danbooru.ja_to_danbooru import jatags_to_danbooru_tags
 import wrapt_timeout_decorator
+from pathlib import Path
 
 
 llm_models_dir = "./llm_models"
@@ -53,6 +54,16 @@ llm_models = {
     "Trinas_Nectar-8B-model_stock.i1-Q4_K_M.gguf": ["mradermacher/Trinas_Nectar-8B-model_stock-i1-GGUF", MessagesFormatterType.MISTRAL],
     "ChatWaifu_v1.4.Q5_K_M.gguf": ["mradermacher/ChatWaifu_v1.4-GGUF", MessagesFormatterType.MISTRAL],
     "ChatWaifu_v1.3.1.Q4_K_M.gguf": ["mradermacher/ChatWaifu_v1.3.1-GGUF", MessagesFormatterType.MISTRAL],
+    "experiment_x-wip-q4_k_m.gguf": ["DreadPoor/EXPERIMENT_X-WIP-Q4_K_M-GGUF", MessagesFormatterType.MISTRAL],
+    "narrathinker12b-q4_k_m.gguf": ["ClaudioItaly/NarraThinker12B-Q4_K_M-GGUF", MessagesFormatterType.MISTRAL],
+    "llama-3.1-8b-matrix-q5_k_m.gguf": ["bunnycore/LLama-3.1-8B-Matrix-Q5_K_M-GGUF", MessagesFormatterType.LLAMA_3],
+    "Barcenas-8b-Cartas.Q5_K_M.gguf": ["mradermacher/Barcenas-8b-Cartas-GGUF", MessagesFormatterType.LLAMA_3],
+    "HannaOpenHermes-2.5-Mistral-7B.Q5_K_M.gguf": ["mradermacher/HannaOpenHermes-2.5-Mistral-7B-GGUF", MessagesFormatterType.MISTRAL],
+    "IceDrinkNameGoesHereRP-7b-Model_Stock.i1-Q4_K_M.gguf": ["mradermacher/IceDrinkNameGoesHereRP-7b-Model_Stock-i1-GGUF", MessagesFormatterType.ALPACA],
+    "Llama-3.1-Literotica-8B.Q4_K_S.gguf": ["mradermacher/Llama-3.1-Literotica-8B-GGUF", MessagesFormatterType.LLAMA_3],
+    "project-12-q4_k_m.gguf": ["ClaudioItaly/Project-12-Q4_K_M-GGUF", MessagesFormatterType.MISTRAL],
+    "L3.1-Celestial-Stone-2x8B.i1-Q4_K_M.gguf": ["mradermacher/L3.1-Celestial-Stone-2x8B-i1-GGUF", MessagesFormatterType.LLAMA_3],
+    "experiment_y-wip-q4_k_m.gguf": ["DreadPoor/EXPERIMENT_Y-WIP-Q4_K_M-GGUF", MessagesFormatterType.LLAMA_3],
     "L3.1-Vulca-Epith-Bluegrade-v0.2-8B.q8_0.gguf": ["kromquant/L3.1-Vulca-Epith-Bluegrade-v0.2-8B-GGUFs", MessagesFormatterType.LLAMA_3],
     "llama-3.1-8b-omnimatrix-iq4_nl-imat.gguf": ["bunnycore/Llama-3.1-8B-OmniMatrix-IQ4_NL-GGUF", MessagesFormatterType.LLAMA_3],
     "L3.1-Artemis-d-8B.i1-Q5_K_M.gguf": ["mradermacher/L3.1-Artemis-d-8B-i1-GGUF", MessagesFormatterType.LLAMA_3],
@@ -1087,3 +1098,75 @@ def dolphin_parse_simple(
     else:
         prompts = list_uniq(to_list(raw_prompt) + ["nsfw", "explicit", "rating_explicit"])
     return ", ".join(prompts)
+
+
+# https://huggingface.co/spaces/CaioXapelaum/GGUF-Playground
+import cv2
+cv2.setNumThreads(1)
+
+@spaces.GPU()
+def respond_playground(
+    message,
+    history: list[tuple[str, str]],
+    model,
+    system_message,
+    max_tokens,
+    temperature,
+    top_p,
+    top_k,
+    repeat_penalty,
+):
+    if override_llm_format:
+        chat_template = override_llm_format
+    else:
+        chat_template = llm_models[model][1]
+
+    llm = Llama(
+        model_path=str(Path(f"{llm_models_dir}/{model}")),
+        flash_attn=True,
+        n_gpu_layers=81, # 81
+        n_batch=1024,
+        n_ctx=8192, #8192
+    )
+    provider = LlamaCppPythonProvider(llm)
+
+    agent = LlamaCppAgent(
+        provider,
+        system_prompt=f"{system_message}",
+        predefined_messages_formatter_type=chat_template,
+        debug_output=False
+    )
+    
+    settings = provider.get_provider_default_settings()
+    settings.temperature = temperature
+    settings.top_k = top_k
+    settings.top_p = top_p
+    settings.max_tokens = max_tokens
+    settings.repeat_penalty = repeat_penalty
+    settings.stream = True
+
+    messages = BasicChatHistory()
+
+    # Add user and assistant messages to the history
+    for msn in history:
+        user = {'role': Roles.user, 'content': msn[0]}
+        assistant = {'role': Roles.assistant, 'content': msn[1]}
+        messages.add_message(user)
+        messages.add_message(assistant)
+
+    # Stream the response
+    try:
+        stream = agent.get_chat_response(
+            message,
+            llm_sampling_settings=settings,
+            chat_history=messages,
+            returns_streaming_generator=True,
+            print_output=False
+        )
+
+        outputs = ""
+        for output in stream:
+            outputs += output
+            yield outputs
+    except Exception as e:
+        yield f"Error during response generation: {str(e)}"

requirements.txt

@@ -15,4 +15,5 @@ httpcore
 googletrans==4.0.0rc1
 git+https://github.com/huggingface/diffusers
 rapidfuzz
-wrapt-timeout-decorator
+wrapt-timeout-decorator
+opencv-python