pseudotensor
commited on
Commit
•
82ea7ed
1
Parent(s):
5354396
Update README.md
Browse files
README.md
CHANGED
@@ -5,16 +5,16 @@ Inference
|
|
5 |
from transformers import AutoTokenizer, AutoModelForCausalLM
|
6 |
import torch
|
7 |
device = torch.device("cuda:0")
|
8 |
-
model_info = "
|
9 |
tokenizer = AutoTokenizer.from_pretrained(model_info, trust_remote_code=True)
|
10 |
model = AutoModelForCausalLM.from_pretrained(model_info, trust_remote_code=True, torch_dtype=torch.bfloat16)
|
11 |
model.eval()
|
12 |
model.to(device)
|
13 |
-
text = "
|
14 |
from predict import predict
|
15 |
out = predict(model, text, tokenizer=tokenizer, max_gen_len=200, top_p=0.95,
|
16 |
seed=1234, topk=100, temperature=0.9, sft=True, device=device,
|
17 |
-
model_name="
|
18 |
print(out)
|
19 |
```
|
20 |
License
|
|
|
5 |
from transformers import AutoTokenizer, AutoModelForCausalLM
|
6 |
import torch
|
7 |
device = torch.device("cuda:0")
|
8 |
+
model_info = "h2oai/h2ogpt-16k-aquilachat2-34b"
|
9 |
tokenizer = AutoTokenizer.from_pretrained(model_info, trust_remote_code=True)
|
10 |
model = AutoModelForCausalLM.from_pretrained(model_info, trust_remote_code=True, torch_dtype=torch.bfloat16)
|
11 |
model.eval()
|
12 |
model.to(device)
|
13 |
+
text = "Who are you?"
|
14 |
from predict import predict
|
15 |
out = predict(model, text, tokenizer=tokenizer, max_gen_len=200, top_p=0.95,
|
16 |
seed=1234, topk=100, temperature=0.9, sft=True, device=device,
|
17 |
+
model_name="h2oai/h2ogpt-16k-aquilachat2-34b")
|
18 |
print(out)
|
19 |
```
|
20 |
License
|