Upload folder using huggingface_hub

README.md

@@ -0,0 +1,58 @@
+---
+tags:
+- finetuned
+- quantized
+- 4-bit
+- AWQ
+- text-generation
+- mixtral
+model_name: Mixtral-8x22B-Instruct-v0.1-AWQ
+base_model: mistralai/Mixtral-8x22B-Instruct-v0.1
+inference: false
+model_creator: mistralai
+pipeline_tag: text-generation
+quantized_by: MaziyarPanahi
+---
+# Description
+[MaziyarPanahi/Mixtral-8x22B-Instruct-v0.1-AWQ](https://huggingface.co/MaziyarPanahi/Mixtral-8x22B-Instruct-v0.1-AWQ) is a quantized (AWQ) version of [mistralai/Mixtral-8x22B-Instruct-v0.1](https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1)
+
+## How to use
+### Install the necessary packages
+
+```
+pip install --upgrade accelerate autoawq transformers
+```
+
+### Example Python code
+
+
+```python
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+model_id = "MaziyarPanahi/Mixtral-8x22B-Instruct-v0.1-AWQ"
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+model = AutoModelForCausalLM.from_pretrained(model_id).to(0)
+
+text = "User:\nHello can you provide me with top-3 cool places to visit in Paris?\n\nAssistant:\n"
+inputs = tokenizer(text, return_tensors="pt").to(0)
+
+out = model.generate(**inputs, max_new_tokens=300)
+print(tokenizer.decode(out[0], skip_special_tokens=True))
+```
+
+Results:
+```
+User:
+Hello can you provide me with top-3 cool places to visit in Paris?
+
+Assistant:
+Absolutely, here are my top-3 recommendations for must-see places in Paris:
+
+1. The Eiffel Tower: An icon of Paris, this wrought-iron lattice tower is a global cultural icon of France and is among the most recognizable structures in the world. Climbing up to the top offers breathtaking views of the city.
+
+2. The Louvre Museum: Home to thousands of works of art, the Louvre is the world's largest art museum and a historic monument in Paris. Must-see pieces include the Mona Lisa, the Winged Victory of Samothrace, and the Venus de Milo.
+
+3. Notre-Dame Cathedral: This cathedral is a masterpiece of French Gothic architecture and is famous for its intricate stone carvings, beautiful stained glass, and its iconic twin towers. Be sure to spend some time exploring its history and learning about the fascinating restoration efforts post the 2019 fire.
+
+I hope you find these recommendations helpful and that they make for an enjoyable and memorable trip to Paris. Safe travels!
+```

config.json

@@ -0,0 +1,40 @@
+{
+  "_name_or_path": "/home/maziyar/.cache/huggingface/hub/models--mistralai--Mixtral-8x22B-Instruct-v0.1/snapshots/514d47be2925f7b5d845bbb34a29d5c73ccf53d8",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 6144,
+  "initializer_range": 0.02,
+  "intermediate_size": 16384,
+  "max_position_embeddings": 65536,
+  "model_type": "mixtral",
+  "num_attention_heads": 48,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 56,
+  "num_key_value_heads": 8,
+  "num_local_experts": 8,
+  "output_router_logits": false,
+  "quantization_config": {
+    "bits": 4,
+    "group_size": 128,
+    "modules_to_not_convert": [
+      "gate"
+    ],
+    "quant_method": "awq",
+    "version": "gemm",
+    "zero_point": true
+  },
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "router_aux_loss_coef": 0.001,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.38.2",
+  "use_cache": true,
+  "vocab_size": 32768
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "do_sample": true,
+  "eos_token_id": 2,
+  "transformers_version": "4.38.2"
+}

model-00001-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39b3d5123fdd10549e05345589da529fe3289aaa89bdedd9fda0f346bc85270e
+size 4979210376

model-00002-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8975d56c263b844b251b78b7e9e6d31a7a000ee8d2eb6292d1738c1860b1de86
+size 4994966792

model-00003-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1ad0647dfa2edda601269fd88756421542e248f1aa20397111d44c880912dc1
+size 4994966904

model-00004-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a28a3abf90fde50f250188a43364cc5500afca9b2283060fdb05407ed8a6835f
+size 4994967128

model-00005-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72f13c3119f6c2352747048f9596b8e0dbb11cc4578ba220f4c6a5f8513f6242
+size 4999807128

model-00006-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b1786bfbe62a4ea1866da3507245664a1acfea14b31a196c1d1154bba06f791
+size 4996540120

model-00007-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:758073c1000379218375c00d9ede32ed6c029662e96ba5f78e656e65488e0807
+size 4994967128

model-00008-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36048e0a2d701ced330ac6bddeaca10472e1a502d5990d21718f2d424b275fc5
+size 4994967128

model-00009-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd5f9dae0bd579b0523521e55630ca074d13b9c9de6be533fa04e9e3f78bd0bb
+size 4994967128

model-00010-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9527ead46bffe530ad605d701b5442c3615a95aa5ed75392fe3c7141f42eb0e3
+size 4994967128

model-00011-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10a3b3a9e3ec10c95f455186fb04cf6c86318e5dd62f10b1758f8b528611fa24
+size 4999807128

model-00012-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:229f6b4f4448a8bd1be5308423ae82b273a9e8ee35d7cc0e4792964d1e10875b
+size 4996540120

model-00013-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b559a2f305f4b1c57b40447f34a77c6741856fe6903e8dfcb8298ab1f5a4b27
+size 4994967128

model-00014-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1720f6f0911204d4da80526ee782bad2b8a8890b0ed11603f2aecda771836ce0
+size 4994967128

model-00015-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd33801de8d34b5ef5dcd2d3548c5ca4ebb4a6295b47f679514de041efefd458
+size 3736943864

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,99 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[INST]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[/INST]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "[TOOL_CALLS]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "[AVAILABLE_TOOLS]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "[/AVAILABLE_TOOLS]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "[TOOL_RESULTS]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "[/TOOL_RESULTS]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{{bos_token}}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ ' [INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + message['content'] + ' ' + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}