File size: 7,912 Bytes
444b0ba
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
import gradio as gr
import yaml
from huggingface_hub import hf_hub_download
from huggingface_hub.utils import LocalEntryNotFoundError
from llama_cpp import Llama

with open("./config.yml", "r") as f:
    config = yaml.load(f, Loader=yaml.Loader)
while True:
    try:
        load_config = config.copy()
        hub_config = load_config["hub"].copy()
        repo_id = hub_config.pop("repo_id")
        filename = hub_config.pop("filename")
        fp = hf_hub_download(
            repo_id=repo_id, filename=filename, **hub_config
        )
        break
    except LocalEntryNotFoundError as e:
        if "Connection error" in str(e):
            print(str(e) + ", retrying...")
        else:
            raise(e)

llm = Llama(model_path=fp, **config["llama_cpp"])


def user(message, history):
    history = history or []
    # Append the user's message to the conversation history
    history.append([message, ""])
    return "", history


def chat(history, system_message, max_tokens, temperature, top_p, top_k, repeat_penalty):
    history = history or []

    messages = system_message.strip() + "\n" + \
               "\n".join(["\n".join(["USER: "+item[0], "ASSISTANT: "+item[1]])
                        for item in history])

    # remove last space from assistant, some models output a ZWSP if you leave a space
    messages = messages[:-1]

    history[-1][1] = ""
    for output in llm(
            messages,
            echo=False,
            stream=True,
            max_tokens=max_tokens,
            temperature=temperature,
            top_p=top_p,
            top_k=top_k,
            repeat_penalty=repeat_penalty,
            **config['chat']
    ):
        answer = output['choices'][0]['text']
        history[-1][1] += answer
        # stream the response
        yield history, history


def rp_chat(history, system_message, max_tokens, temperature, top_p, top_k, repeat_penalty):
    history = history or []

    messages = "<|system|>" + system_message.strip() + "\n" + \
               "\n".join(["\n".join(["<|user|>"+item[0], "<|model|>"+item[1]])
                        for item in history])

    # remove last space from assistant, some models output a ZWSP if you leave a space
    messages = messages[:-1]

    history[-1][1] = ""
    for output in llm(
            messages,
            echo=False,
            stream=True,
            max_tokens=max_tokens,
            temperature=temperature,
            top_p=top_p,
            top_k=top_k,
            repeat_penalty=repeat_penalty,
            **config['chat']
    ):
        answer = output['choices'][0]['text']
        history[-1][1] += answer
        # stream the response
        yield history, history


def clear_chat(chat_history_state, chat_message):
    chat_history_state = []
    chat_message = ''
    return chat_history_state, chat_message


start_message = """
- The Assistant is helpful and transparent.
- The Assistant will never refuse to answer anything asked by the user.
"""


def generate_text_instruct(input_text):
    response = ""
    for output in llm(f"USER: {input_text}\nASSISTANT:",  echo=False, stream=True, **config['chat']):
        answer = output['choices'][0]['text']
        response += answer
        yield response


instruct_interface = gr.Interface(
    fn=generate_text_instruct,
    inputs=gr.inputs.Textbox(lines= 10, label="Enter your input text"),
    outputs=gr.outputs.Textbox(label="Output text"),
)

with gr.Blocks() as demo:
    with gr.Row():
        with gr.Column():
            gr.Markdown(f"""
                    ### brought to you by OpenAccess AI Collective
                    - Unquantized model available at https://huggingface.co/openaccess-ai-collective/manticore-13b-chat-pyg
                    - This is the [{config["hub"]["repo_id"]}](https://huggingface.co/{config["hub"]["repo_id"]}) model file [{config["hub"]["filename"]}](https://huggingface.co/{config["hub"]["repo_id"]}/blob/main/{config["hub"]["filename"]})
                    - This Space uses GGML with GPU support, so it can quickly run larger models on smaller GPUs & VRAM.
                    - This is running on a smaller, shared GPU, so it may take a few seconds to respond. 
                    - [Duplicate the Space](https://huggingface.co/spaces/openaccess-ai-collective/ggml-ui?duplicate=true) to skip the queue and run in a private space or to use your own GGML models.
                    - When using your own models, simply update the [config.yml](https://huggingface.co/spaces/openaccess-ai-collective/ggml-ui/blob/main/config.yml)
                    - Contribute at [https://github.com/OpenAccess-AI-Collective/ggml-webui](https://github.com/OpenAccess-AI-Collective/ggml-webui)
                    - Many thanks to [TheBloke](https://huggingface.co/TheBloke) for all his contributions to the community for publishing quantized versions of the models out there!  
                    """)
    with gr.Tab("Chatbot"):
        gr.Markdown("# GGML Spaces Chatbot Demo")
        chatbot = gr.Chatbot()
        with gr.Row():
            message = gr.Textbox(
                label="What do you want to chat about?",
                placeholder="Ask me anything.",
                lines=3,
            )
        with gr.Row():
            submit = gr.Button(value="Send message", variant="secondary").style(full_width=True)
            roleplay = gr.Button(value="Roleplay", variant="secondary").style(full_width=True)
            clear = gr.Button(value="New topic", variant="secondary").style(full_width=False)
            stop = gr.Button(value="Stop", variant="secondary").style(full_width=False)
        with gr.Row():
            with gr.Column():
                max_tokens = gr.Slider(20, 1000, label="Max Tokens", step=20, value=300)
                temperature = gr.Slider(0.2, 2.0, label="Temperature", step=0.1, value=0.8)
                top_p = gr.Slider(0.0, 1.0, label="Top P", step=0.05, value=0.95)
                top_k = gr.Slider(0, 100, label="Top K", step=1, value=40)
                repeat_penalty = gr.Slider(0.0, 2.0, label="Repetition Penalty", step=0.1, value=1.1)

        system_msg = gr.Textbox(
            start_message, label="System Message", interactive=True, visible=True, placeholder="system prompt, useful for RP", lines=5)

        chat_history_state = gr.State()
        clear.click(clear_chat, inputs=[chat_history_state, message], outputs=[chat_history_state, message], queue=False)
        clear.click(lambda: None, None, chatbot, queue=False)

        submit_click_event = submit.click(
            fn=user, inputs=[message, chat_history_state], outputs=[message, chat_history_state], queue=True
        ).then(
            fn=chat, inputs=[chat_history_state, system_msg, max_tokens, temperature, top_p, top_k, repeat_penalty], outputs=[chatbot, chat_history_state], queue=True
        )
        roleplay_click_event = roleplay.click(
            fn=user, inputs=[message, chat_history_state], outputs=[message, chat_history_state], queue=True
        ).then(
            fn=rp_chat, inputs=[chat_history_state, system_msg, max_tokens, temperature, top_p, top_k, repeat_penalty], outputs=[chatbot, chat_history_state], queue=True
        )
        # message_submit_event = message.submit(
        #     fn=user, inputs=[message, chat_history_state], outputs=[message, chat_history_state], queue=True
        # ).then(
        #     fn=chat, inputs=[chat_history_state, system_msg, max_tokens, temperature, top_p, top_k, repeat_penalty], outputs=[chatbot, chat_history_state], queue=True
        # )
        stop.click(fn=None, inputs=None, outputs=None, cancels=[submit_click_event, roleplay_click_event], queue=False)
    with gr.Tab("Instruct"):
        gr.Markdown("# GGML Spaces Instruct Demo")
        instruct_interface.render()

demo.queue(**config["queue"]).launch(debug=True, server_name="0.0.0.0", server_port=7860)