mlc-ai
/

Llama-3.1-70B-Instruct-q4f16_1-MLC

Model card Files Files and versions Community

ruihanglai commited on about 20 hours ago

Commit

e1581d5

•

1 Parent(s): 5c92034

Update mlc-chat-config.json

Files changed (1) hide show

mlc-chat-config.json +3 -3

mlc-chat-config.json CHANGED Viewed

@@ -19,16 +19,16 @@
       "rope_type": "llama3"
     },
     "context_window_size": 131072,
-    "prefill_chunk_size": 2048,
     "num_key_value_heads": 8,
     "head_dim": 128,
     "tensor_parallel_shards": 1,
-    "max_batch_size": 80
   },
   "vocab_size": 128256,
   "context_window_size": 131072,
   "sliding_window_size": -1,
-  "prefill_chunk_size": 2048,
   "attention_sink_size": -1,
   "tensor_parallel_shards": 1,
   "temperature": 0.6,

       "rope_type": "llama3"
     },
     "context_window_size": 131072,
+    "prefill_chunk_size": 8192,
     "num_key_value_heads": 8,
     "head_dim": 128,
     "tensor_parallel_shards": 1,
+    "max_batch_size": 128
   },
   "vocab_size": 128256,
   "context_window_size": 131072,
   "sliding_window_size": -1,
+  "prefill_chunk_size": 8192,
   "attention_sink_size": -1,
   "tensor_parallel_shards": 1,
   "temperature": 0.6,