Spaces:

John6666
/

text2tag-llm

Running on Zero

John6666 commited on Aug 22

Commit

7d22dbb

•

1 Parent(s): 3c9dc34

Upload llmdolphin.py

Files changed (1) hide show

llmdolphin.py CHANGED Viewed

@@ -692,7 +692,7 @@ def dolphin_respond(
     llm = Llama(
         model_path=str(Path(f"{llm_models_dir}/{model}")),
         flash_attn=True,
-        n_gpu_layers=40, # 81
         n_batch=1024,
         n_ctx=4096, #8192
         n_threads=8,
@@ -787,7 +787,7 @@ def dolphin_respond_auto(
     llm = Llama(
         model_path=str(Path(f"{llm_models_dir}/{model}")),
         flash_attn=True,
-        n_gpu_layers=40, # 81
         n_batch=1024,
         n_ctx=4096, #8192
         n_threads=8,

     llm = Llama(
         model_path=str(Path(f"{llm_models_dir}/{model}")),
         flash_attn=True,
+        n_gpu_layers=81, # 81
         n_batch=1024,
         n_ctx=4096, #8192
         n_threads=8,
     llm = Llama(
         model_path=str(Path(f"{llm_models_dir}/{model}")),
         flash_attn=True,
+        n_gpu_layers=81, # 81
         n_batch=1024,
         n_ctx=4096, #8192
         n_threads=8,