Initial commit

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+model.onnx_data filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

3rd_party_licenses.txt

@@ -0,0 +1,48 @@
+1: Name: TensorRT Model Optimizer - Windows
+Version: 0.19
+LicenseText: MIT License
+
+Copyright (c) 2023 MIT HAN Lab
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
+2: Name: onnxruntime-genai-directml
+Version: 0.4.0
+LicenseText:     MIT License
+
+    Copyright (c) Microsoft Corporation.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE

License.txt

@@ -0,0 +1 @@
+GOVERNING TERMS: Use of this model is governed by the NVIDIA Open Model License Agreement (found at https://developer.download.nvidia.com/licenses/nvidia-open-model-license-agreement-june-2024.pdf). ADDITIONAL INFORMATION: Apache License, Version 2.0 (found at https://huggingface.co/datasets/choosealicense/licenses/blob/main/markdown/apache-2.0.md).

Readme.txt

@@ -0,0 +1,10 @@
+To run inference with this model, please follow below steps -
+
+1. Install Nvidia Graphics driver R565 or higher. 
+2. Install python 3.10.11.
+3. Setup GenAI DirectML via
+    a. Install numpy - pip install numpy==2.1.0
+    b. Install GenAI - pip install onnxruntime-genai-directml==0.5.0
+4. Download inference script - curl https://github.com/microsoft/onnxruntime-genai/blob/rel-0.5.0/examples/python/phi3-qa.py -o phi3-qa.py
+5. Run inference - python phi3-qa.py -m <model-downloaded-path>
+    a. Enter prompt - "What is GenAI?"

genai_config.json

@@ -0,0 +1,54 @@
+{
+    "model": {
+        "bos_token_id": 1,
+        "context_length": 1024000,
+        "decoder": {
+            "session_options": {
+                "log_id": "onnxruntime-genai",
+                "provider_options": [
+                    {
+                        "dml": {}
+                    }
+                ]
+            },
+            "filename": "model.onnx",
+            "head_size": 128,
+            "hidden_size": 5120,
+            "inputs": {
+                "input_ids": "input_ids",
+                "attention_mask": "attention_mask",
+                "position_ids": "position_ids",
+                "past_key_names": "past_key_values.%d.key",
+                "past_value_names": "past_key_values.%d.value"
+            },
+            "outputs": {
+                "logits": "logits",
+                "present_key_names": "present.%d.key",
+                "present_value_names": "present.%d.value"
+            },
+            "num_attention_heads": 32,
+            "num_hidden_layers": 40,
+            "num_key_value_heads": 8
+        },
+        "eos_token_id": 2,
+        "pad_token_id": 2,
+        "type": "mistral",
+        "vocab_size": 131072
+    },
+    "search": {
+        "diversity_penalty": 0.0,
+        "do_sample": false,
+        "early_stopping": true,
+        "length_penalty": 1.0,
+        "max_length": 1024000,
+        "min_length": 0,
+        "no_repeat_ngram_size": 0,
+        "num_beams": 1,
+        "num_return_sequences": 1,
+        "past_present_share_buffer": true,
+        "repetition_penalty": 1.0,
+        "temperature": 1.0,
+        "top_k": 1,
+        "top_p": 1.0
+    }
+}

model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:219f817360446debb5e2b557f2c2e40dc6a5cfc2c9a132ede25b9278b1344372
+size 545151

model.onnx_data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ef8e2c8550824c282e318d9b074d4c93d054282c37d6493f03015ee825a2b36
+size 8573839360

quantization_log_and_pip_list.txt

@@ -0,0 +1,184 @@
+
+(venv2) D:\modelopt-windows-scripts\ONNX_PTQ>python D:\opset21_patrice.py --onnx_path="D:\GenAI\models\FP16_Mistral-Nemo-Instruct-2407_ONNX\model.onnx" --output_path="D:\GenAI\models\FP16_Mistral-Nemo-Instruct-2407_ONNX\opset_21\model.onnx"
+Printing opset info of given input model...
+
+Domain:
+  Version: 14
+
+Domain: com.microsoft
+  Version: 1
+
+Printing opset info of output model...
+
+Domain:
+  Version: 21
+
+Domain: com.microsoft
+  Version: 1
+
+
+(venv2) D:\modelopt-windows-scripts\ONNX_PTQ>python quantize_script.py --model_name=mistralai/Mistral-Nemo-Instruct-2407  --onnx_path=D:\GenAI\models\FP16_Mistral-Nemo-Instruct-2407_ONNX\opset_21\model.onnx --output_path="D:\GenAI\models\FP16_Mistral-Nemo-Instruct-2407_ONNX\opset_21\default_quant_cuda_ep_calib\model.onnx" --calibration_eps=cuda
+
+--Quantize-Script-- algo=awq_lite, dataset=cnn, calib_size=32, batch_size=1, block_size=128, add-position-ids=True, past-kv=True, rcalib=False, device=cpu, use_zero_point=False
+
+
+
+--Quantize-Script-- awqlite_alpha_step=0.1, awqlite_fuse_nodes=False, awqlite_run_per_subgraph=False, awqclip_alpha_step=0.05, awqclip_alpha_min=0.5, awqclip_bsz_col=1024, calibration_eps=['cuda']
+
+D:\venv2\Lib\site-packages\transformers\models\auto\configuration_auto.py:1002: FutureWarning: The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.
+  warnings.warn(
+D:\venv2\Lib\site-packages\transformers\models\auto\tokenization_auto.py:809: FutureWarning: The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.
+  warnings.warn(
+
+--Quantize-Script-- number_of_batched_samples=32, batch-input-ids-list-len=32, batched_attention_mask=32
+
+
+--Quantize-Script-- number of batched inputs = 32
+
+INFO:root:
+Quantizing the model....
+
+INFO:root:Quantization Mode: int4
+INFO:root:Finding quantizable weights and augmenting graph output with input activations
+INFO:root:Augmenting took 0.031656503677368164 seconds
+INFO:root:Saving the model took 60.20284128189087 seconds
+2024-11-05 22:37:34.5783341 [W:onnxruntime:, transformer_memcpy.cc:74 onnxruntime::MemcpyTransformer::ApplyImpl] 11 Memcpy nodes are added to the graph main_graph for CUDAExecutionProvider. It might have negative impact on performance (including unable to run CUDA graph). Set session_options.log_severity_level=1 to see the detail logs before this message.
+2024-11-05 22:37:34.5949880 [W:onnxruntime:, session_state.cc:1168 onnxruntime::VerifyEachNodeIsAssignedToAnEp] Some nodes were not assigned to the preferred execution providers which may or may not have an negative impact on performance. e.g. ORT explicitly assigns shape related ops to CPU to improve perf.
+2024-11-05 22:37:34.6026375 [W:onnxruntime:, session_state.cc:1170 onnxruntime::VerifyEachNodeIsAssignedToAnEp] Rerunning with verbose output on a non-minimal build will show node assignments.
+Getting activation names maps...: 100%|██████████████████████████████████████████████████████| 280/280 [00:00<?, ?it/s]
+Running AWQ scale search per node...: 100%|██████████████████████████████████████████| 280/280 [17:50<00:00,  3.82s/it]
+INFO:root:AWQ scale search took 1070.4731740951538 seconds
+Quantizing the weights...: 100%|█████████████████████████████████████████████████████| 280/280 [00:15<00:00, 17.78it/s]
+INFO:root:Quantizing actual weights took 15.744078636169434 seconds
+INFO:root:Inserting DQ nodes and input_pre_quant_scale node using quantized weights and scales ...
+INFO:root:Inserting nodes took 0.17318105697631836 seconds
+INFO:root:Exporting the quantized graph ...
+Loading extension modelopt_round_and_pack_ext...
+
+INFO:root:Exporting took 59.45134162902832 seconds
+INFO:root:
+Quantization process took 1223.9775414466858 seconds
+INFO:root:Saving to D:\GenAI\models\FP16_Mistral-Nemo-Instruct-2407_ONNX\opset_21\default_quant_cuda_ep_calib\model.onnx took 9.476586818695068 seconds
+
+Done
+
+
+(venv2) D:\modelopt-windows-scripts\ONNX_PTQ>pip list
+Package              Version
+-------------------- -------------------------
+aiohappyeyeballs     2.4.3
+aiohttp              3.10.10
+aiosignal            1.3.1
+annotated-types      0.7.0
+attrs                24.2.0
+certifi              2024.8.30
+charset-normalizer   3.4.0
+cloudpickle          3.1.0
+colorama             0.4.6
+coloredlogs          15.0.1
+cppimport            22.8.2
+cupy-cuda12x         13.3.0
+datasets             3.1.0
+dill                 0.3.8
+fastrlock            0.8.2
+filelock             3.16.1
+flatbuffers          24.3.25
+frozenlist           1.5.0
+fsspec               2024.9.0
+huggingface-hub      0.26.2
+humanfriendly        10.0
+idna                 3.10
+Jinja2               3.1.4
+Mako                 1.3.6
+markdown-it-py       3.0.0
+MarkupSafe           3.0.2
+mdurl                0.1.2
+mpmath               1.3.0
+multidict            6.1.0
+multiprocess         0.70.16
+networkx             3.4.2
+ninja                1.11.1.1
+numpy                1.26.4
+nvidia-modelopt      0.20.1.dev20+g299b7f8a098
+onnx                 1.16.0
+onnx-graphsurgeon    0.5.2
+onnxconverter-common 1.14.0
+onnxmltools          1.12.0
+onnxruntime-gpu      1.20.0
+packaging            24.1
+pandas               2.2.3
+pip                  24.0
+propcache            0.2.0
+protobuf             3.20.2
+pyarrow              18.0.0
+pybind11             2.13.6
+pydantic             2.9.2
+pydantic_core        2.23.4
+Pygments             2.18.0
+pyreadline3          3.5.4
+python-dateutil      2.9.0.post0
+pytz                 2024.2
+PyYAML               6.0.2
+regex                2024.9.11
+requests             2.32.3
+rich                 13.9.4
+safetensors          0.4.5
+scipy                1.14.1
+setuptools           65.5.0
+six                  1.16.0
+sympy                1.13.3
+tokenizers           0.20.2
+torch                2.4.0
+tqdm                 4.66.6
+transformers         4.46.1
+typing_extensions    4.12.2
+tzdata               2024.2
+urllib3              2.2.3
+xxhash               3.5.0
+yarl                 1.17.1
+
+[notice] A new release of pip is available: 24.0 -> 24.3.1
+[notice] To update, run: python.exe -m pip install --upgrade pip
+
+(venv2) D:\modelopt-windows-scripts\ONNX_PTQ>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0240ce510f08e6c2041724e9043e33be9d251d1e4a4d94eb68cd47b954b61d2
+size 17078292