Lam-Hung commited on
Commit
f401051
1 Parent(s): 3ae0fea

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +135 -51
app.py CHANGED
@@ -1,63 +1,147 @@
 
 
 
 
1
  import gradio as gr
2
- from huggingface_hub import InferenceClient
3
-
4
- """
5
- For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
6
- """
7
- client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
8
-
9
-
10
- def respond(
11
- message,
12
- history: list[tuple[str, str]],
13
- system_message,
14
- max_tokens,
15
- temperature,
16
- top_p,
17
- ):
18
- messages = [{"role": "system", "content": system_message}]
19
-
20
- for val in history:
21
- if val[0]:
22
- messages.append({"role": "user", "content": val[0]})
23
- if val[1]:
24
- messages.append({"role": "assistant", "content": val[1]})
25
-
26
- messages.append({"role": "user", "content": message})
27
-
28
- response = ""
29
-
30
- for message in client.chat_completion(
31
- messages,
32
- max_tokens=max_tokens,
33
- stream=True,
34
- temperature=temperature,
35
- top_p=top_p,
36
- ):
37
- token = message.choices[0].delta.content
38
-
39
- response += token
40
- yield response
41
-
42
- """
43
- For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
44
- """
45
- demo = gr.ChatInterface(
46
- respond,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
47
  additional_inputs=[
48
- gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
49
- gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
50
- gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
51
  gr.Slider(
 
 
 
 
 
 
 
 
52
  minimum=0.1,
 
 
 
 
 
 
 
53
  maximum=1.0,
54
- value=0.95,
55
  step=0.05,
56
- label="Top-p (nucleus sampling)",
57
  ),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
58
  ],
 
 
 
 
 
 
 
 
59
  )
60
 
61
 
62
  if __name__ == "__main__":
63
- demo.launch()
 
1
+ import os
2
+ from threading import Thread
3
+ from typing import Iterator
4
+
5
  import gradio as gr
6
+ import spaces
7
+ import torch
8
+ from transformers import BitsAndBytesConfig, AutoConfig, AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
9
+
10
+
11
+ MAX_MAX_NEW_TOKENS = 2048
12
+ DEFAULT_MAX_NEW_TOKENS = 1024
13
+ MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "4096"))
14
+
15
+ device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
16
+ model_path = "vinai/PhoGPT-4B-Chat"
17
+
18
+ config = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
19
+ config.init_device = device
20
+
21
+ quantization = BitsAndBytesConfig(load_in_8bit=True)
22
+
23
+ model = AutoModelForCausalLM.from_pretrained(model_path,
24
+ config=config,
25
+ quantization_config =quantization,
26
+ torch_dtype=torch.bfloat16,
27
+ trust_remote_code=True,
28
+ low_cpu_mem_usage=True)
29
+
30
+ model.eval()
31
+
32
+ tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
33
+
34
+
35
+ @spaces.GPU(duration=120)
36
+ def generate(
37
+ message: str,
38
+ chat_history: list[tuple[str, str]],
39
+ max_new_tokens: int = 1024,
40
+ temperature: float = 0.6,
41
+ top_p: float = 0.9,
42
+ top_k: int = 50,
43
+ repetition_penalty: float = 1.2,
44
+ ) -> Iterator[str]:
45
+ conversation = []
46
+ for user, assistant in chat_history:
47
+ conversation.extend(
48
+ [
49
+ {"role": "user", "content": user},
50
+ {"role": "assistant", "content": assistant},
51
+ ]
52
+ )
53
+ conversation.append({"role": "user", "content": message})
54
+
55
+ input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt")
56
+ if input_ids.shape[1] > MAX_INPUT_TOKEN_LENGTH:
57
+ input_ids = input_ids[:, -MAX_INPUT_TOKEN_LENGTH:]
58
+ gr.Warning(f"Trimmed input from conversation as it was longer than {MAX_INPUT_TOKEN_LENGTH} tokens.")
59
+ input_ids = input_ids.to(model.device)
60
+
61
+ streamer = TextIteratorStreamer(tokenizer, timeout=20.0, skip_prompt=True, skip_special_tokens=True)
62
+ generate_kwargs ={
63
+ "input_ids": input_ids,
64
+ "streamer":streamer,
65
+ "max_new_tokens":max_new_tokens,
66
+ "do_sample":True,
67
+ "top_p":top_p,
68
+ "top_k":top_k,
69
+ "temperature":temperature,
70
+ "num_beams":1,
71
+ "repetition_penalty":repetition_penalty,
72
+ "eos_token_id":tokenizer.eos_token_id,
73
+ "pad_token_id":tokenizer.pad_token_id
74
+ }
75
+ t = Thread(target=model.generate, kwargs=generate_kwargs)
76
+ t.start()
77
+
78
+ outputs = []
79
+ for text in streamer:
80
+ outputs.append(text)
81
+ yield "".join(outputs)
82
+
83
+
84
+ chat_interface = gr.ChatInterface(
85
+ fn=generate,
86
+ chatbot=gr.Chatbot(height=500, label = "VN GPT", show_label=True),
87
+ textbox=gr.Textbox(placeholder="Nhập hội thoại tại đây", container=False, scale=7),
88
  additional_inputs=[
 
 
 
89
  gr.Slider(
90
+ label="Độ dài token",
91
+ minimum=1,
92
+ maximum=MAX_MAX_NEW_TOKENS,
93
+ step=1,
94
+ value=DEFAULT_MAX_NEW_TOKENS,
95
+ ),
96
+ gr.Slider(
97
+ label="Độ sáng tạo",
98
  minimum=0.1,
99
+ maximum=4.0,
100
+ step=0.1,
101
+ value=0.6,
102
+ ),
103
+ gr.Slider(
104
+ label="Lựa chọn từ dựa trên xác suất tích lũy",
105
+ minimum=0.05,
106
  maximum=1.0,
 
107
  step=0.05,
108
+ value=0.9,
109
  ),
110
+ gr.Slider(
111
+ label="Lựa chọn k từ có xác suất cao nhất",
112
+ minimum=1,
113
+ maximum=1000,
114
+ step=1,
115
+ value=50,
116
+ ),
117
+ gr.Slider(
118
+ label="Phạt lặp lại",
119
+ minimum=1.0,
120
+ maximum=2.0,
121
+ step=0.05,
122
+ value=1.2,
123
+ ),
124
+ ],
125
+ theme="soft",
126
+ stop_btn=None,
127
+ examples = [
128
+ ["Lợi ích của sữa mẹ ?"],
129
+ ["Sữa non là gì ?"],
130
+ ["Trẻ sơ sinh cần ngủ bao nhiêu giờ mỗi ngày?"],
131
+ ["Bao lâu nên cho trẻ sơ sinh bú một lần?"],
132
+ ["Khi nào nên bắt đầu cho trẻ ăn dặm?"],
133
+ ["Làm thế nào để giúp trẻ ngủ ngon vào ban đêm?"]
134
  ],
135
+
136
+ cache_examples=False,
137
+ title = "VN-GPT",
138
+ clear_btn="🗑️ Xóa",
139
+ undo_btn="↩️ Hoàn tác",
140
+ submit_btn="🚀 Gửi",
141
+ retry_btn="🔄 Thử lại",
142
+ additional_inputs_accordion="Tùy chỉnh nâng cao",
143
  )
144
 
145
 
146
  if __name__ == "__main__":
147
+ chat_interface.queue(max_size=20).launch()