upload lazy lora adapter for llama-2 70b hf

README.md

@@ -1,3 +1,160 @@
 ---
 license: llama2
 ---
+
+## Lazy LoRA
+
+### Benefits
+
+0. using the updated [Meta's LLaMA-2 models](https://huggingface.co/meta-llama/Llama-2-70b-hf).
+1. support [4-bit qlora](https://arxiv.org/abs/2305.14314), extreme GPU memory and inference time saving;
+2. comparable MMLU evaluation dataset results, llama2-70b's 68.9% to our 68.21% (-0.69%).
+3. This lazy-lora adapter is based on [Meta's LLaMA-2-70b-hf](https://huggingface.co/meta-llama/Llama-2-70b-hf), and using the [oasst1 dataset](https://huggingface.co/datasets/OpenAssistant/oasst1), following [Guanaco](https://huggingface.co/timdettmers/guanaco-65b).
+
+### Introduction
+Determine the rank of LoRA layers by the singular values of pretrained weight matrices.
+Also, combines:
+1. LoRA: [LORA: LOW-RANK ADAPTATION OF LARGE LANGUAGE MODELS](https://arxiv.org/abs/2106.09685)
+2. Prefix Tuning: [Prefix-Tuning: Optimizing Continuous Prompts for Generation](https://aclanthology.org/2021.acl-long.3
+53/), [P-Tuning v2: Prompt Tuning Can Be Comparable to Fine-tuning Universally Across Scales and Tasks](https://arxiv.or
+g/pdf/2110.07602.pdf)
+3. Prompt Tuning: [The Power of Scale for Parameter-Efficient Prompt Tuning](https://arxiv.org/abs/2104.08691)
+4. LLaMA adapter: [LLaMA-Adapter: Efficient Fine-tuning of Language Models with Zero-init Attention] (https://arxiv.org/abs/2303.16199)
+in one model.
+
+This allows you to perform LoRA (additional low rank adapters inserted to each linear layer), and prompt learning (additional virtual tokens attached to the input and to the attention layers acting as `past_key_values`)
+
+## Usage:
+```python
+import sys
+sys.path.insert(1, '/workspace/asr/peft/src')
+# TODO set this path to the lazy-lora source code path, 
+# or you can install it from source code:
+# TODO, please install lazylora for usage:
+# git clone [email protected]:Xianchao-Wu/peft.git
+# cd peft
+# python setup.py install
+
+from transformers import (AutoTokenizer, 
+	AutoModelForCausalLM, BitsAndBytesConfig)
+from peft import PeftModel, PeftConfig
+import os
+import torch
+
+#import ipdb; ipdb.set_trace()
+cache_dir="/workspace/asr/peft/qlora"
+# TODO set this cache_dir to the path where you 
+# stored (or, want to store) llama2-70b-hf model
+
+lazylora_dir=os.getcwd() 
+# the path that contains 'adapter_config.json' 
+#     and 'adapter_model.bin'
+
+config = PeftConfig.from_pretrained(lazylora_dir)
+
+tokenizer = AutoTokenizer.from_pretrained(
+    config.base_model_name_or_path,
+    cache_dir=cache_dir,
+    use_auth_token=True
+)
+
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type='nf4',
+    bnb_4bit_compute_dtype=torch.bfloat16
+)
+
+model = AutoModelForCausalLM.from_pretrained(
+    config.base_model_name_or_path,
+    quantization_config=bnb_config,
+    device_map="auto",
+    cache_dir=cache_dir,
+    use_auth_token=True
+)
+#model.print_trainable_parameters()
+print(sum(p.numel() for p in model.parameters()))
+# 34,751,127,552 -> half-size of 70B due to 4-bit loading
+
+model = PeftModel.from_pretrained(model, lazylora_dir)
+print('after adding lazy lora parameters:')
+model.print_trainable_parameters()
+# trainable params: 0 || all params: 35,579,442,176 || trainable%: 0.0
+```
+
+## MMLU result:
+
+```json
+{"mmlu_loss": 2.3140328107200987,
+ "mmlu_eval_accuracy_abstract_algebra": 0.36363636363636365,
+ "mmlu_eval_accuracy_high_school_chemistry": 0.5,
+ "mmlu_eval_accuracy_college_physics": 0.45454545454545453,
+ "mmlu_eval_accuracy_international_law": 0.9230769230769231,
+ "mmlu_eval_accuracy_nutrition": 0.696969696969697,
+ "mmlu_eval_accuracy_world_religions": 0.8947368421052632,
+ "mmlu_eval_accuracy_medical_genetics": 1.0,
+ "mmlu_eval_accuracy_high_school_computer_science": 0.6666666666666666,
+ "mmlu_eval_accuracy_anatomy": 0.5,
+ "mmlu_eval_accuracy_sociology": 1.0,
+ "mmlu_eval_accuracy_human_sexuality": 0.5833333333333334,
+ "mmlu_eval_accuracy_high_school_world_history": 0.7307692307692307,
+ "mmlu_eval_accuracy_jurisprudence": 0.7272727272727273,
+ "mmlu_eval_accuracy_high_school_mathematics": 0.2413793103448276,
+ "mmlu_eval_accuracy_college_biology": 0.8125,
+ "mmlu_eval_accuracy_machine_learning": 0.5454545454545454,
+ "mmlu_eval_accuracy_us_foreign_policy": 1.0,
+ "mmlu_eval_accuracy_high_school_microeconomics": 0.7692307692307693,
+ "mmlu_eval_accuracy_high_school_us_history": 1.0,
+ "mmlu_eval_accuracy_security_studies": 0.7777777777777778,
+ "mmlu_eval_accuracy_college_chemistry": 0.25,
+ "mmlu_eval_accuracy_college_computer_science": 0.5454545454545454,
+ "mmlu_eval_accuracy_miscellaneous": 0.7790697674418605,
+ "mmlu_eval_accuracy_professional_accounting": 0.7419354838709677,
+ "mmlu_eval_accuracy_business_ethics": 0.7272727272727273,
+ "mmlu_eval_accuracy_electrical_engineering": 0.5625,
+ "mmlu_eval_accuracy_elementary_mathematics": 0.4878048780487805,
+ "mmlu_eval_accuracy_high_school_biology": 0.71875,
+ "mmlu_eval_accuracy_college_mathematics": 0.45454545454545453,
+ "mmlu_eval_accuracy_high_school_european_history": 0.7777777777777778,
+ "mmlu_eval_accuracy_professional_law": 0.5588235294117647,
+ "mmlu_eval_accuracy_prehistory": 0.8,
+ "mmlu_eval_accuracy_high_school_macroeconomics": 0.7674418604651163,
+ "mmlu_eval_accuracy_formal_logic": 0.42857142857142855,
+ "mmlu_eval_accuracy_philosophy": 0.7941176470588235,
+ "mmlu_eval_accuracy_astronomy": 0.75,
+ "mmlu_eval_accuracy_clinical_knowledge": 0.7586206896551724,
+ "mmlu_eval_accuracy_global_facts": 0.5,
+ "mmlu_eval_accuracy_high_school_government_and_politics": 0.9523809523809523,
+ "mmlu_eval_accuracy_moral_disputes": 0.6842105263157895,
+ "mmlu_eval_accuracy_econometrics": 0.5,
+ "mmlu_eval_accuracy_management": 0.9090909090909091,
+ "mmlu_eval_accuracy_high_school_psychology": 0.9666666666666667,
+ "mmlu_eval_accuracy_high_school_geography": 0.9090909090909091,
+ "mmlu_eval_accuracy_human_aging": 0.6956521739130435,
+ "mmlu_eval_accuracy_logical_fallacies": 0.7222222222222222,
+ "mmlu_eval_accuracy_moral_scenarios": 0.49,
+ "mmlu_eval_accuracy_conceptual_physics": 0.5384615384615384,
+ "mmlu_eval_accuracy_professional_psychology": 0.782608695652174,
+ "mmlu_eval_accuracy_college_medicine": 0.7727272727272727,
+ "mmlu_eval_accuracy_high_school_physics": 0.11764705882352941,
+ "mmlu_eval_accuracy_computer_security": 0.7272727272727273,
+ "mmlu_eval_accuracy_virology": 0.5555555555555556,
+ "mmlu_eval_accuracy_professional_medicine": 0.7741935483870968,
+ "mmlu_eval_accuracy_marketing": 0.96,
+ "mmlu_eval_accuracy_public_relations": 0.6666666666666666,
+ "mmlu_eval_accuracy_high_school_statistics": 0.5652173913043478,
+ "mmlu_eval_accuracy": 0.682100004303323,
+ "epoch": 1.7}
+```
+
+## License and intended use 
+
+This lazy-lora adapter is based on [Meta's LLaMA-2-70b-hf](https://huggingface.co/meta-llama/Llama-2-70b-hf), and using the [oasst1 dataset](https://huggingface.co/datasets/OpenAssistant/oasst1), following [Guanaco](https://huggingface.co/timdettmers/guanaco-65b).
+
+lazy lora adapter weights are available under LLAMA-2 license. Note the use of the lazy lora adapter weights, requires access to the LLaMA model weighs. Lazy lora is based on LLaMA and therefore should be used according to the LLaMA license.
+
+
+## Risks and Biases
+
+The model can produce factually incorrect output, and should not be relied on to produce factually accurate information. The model was trained on various public datasets; it is possible that this model could generate lewd, biased, or otherwise offensive outputs.
+

adapter_config.json

@@ -0,0 +1,596 @@
+{
+  "base_model_name_or_path": "meta-llama/Llama-2-70b-hf",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lazy_lora_weights": true,
+  "is_r_by_svd": true,
+  "is_r_reuse": true,
+  "lazy_lora_alpha": 16.0,
+  "lazy_lora_dropout": 0.05,
+  "lazy_pre_adapter_type": "none",
+  "lazy_pre_lora_alpha": 0.1,
+  "modules_to_save": null,
+  "num_attention_heads": 64,
+  "num_layers": 80,
+  "num_transformer_submodules": 1,
+  "num_virtual_tokens": null,
+  "peft_type": "LAZY_LORA",
+  "prefix_tuning_config": null,
+  "prompt_tuning_config": null,
+  "r": 64,
+  "r_by_module_dict": {
+    "model.layers.0.mlp.down_proj": 44,
+    "model.layers.0.mlp.gate_proj": 30,
+    "model.layers.0.mlp.up_proj": 31,
+    "model.layers.0.self_attn.k_proj": 39,
+    "model.layers.0.self_attn.o_proj": 29,
+    "model.layers.0.self_attn.q_proj": 26,
+    "model.layers.0.self_attn.v_proj": 35,
+    "model.layers.1.mlp.down_proj": 57,
+    "model.layers.1.mlp.gate_proj": 42,
+    "model.layers.1.mlp.up_proj": 46,
+    "model.layers.1.self_attn.k_proj": 62,
+    "model.layers.1.self_attn.o_proj": 36,
+    "model.layers.1.self_attn.q_proj": 41,
+    "model.layers.1.self_attn.v_proj": 34,
+    "model.layers.10.mlp.down_proj": 63,
+    "model.layers.10.mlp.gate_proj": 64,
+    "model.layers.10.mlp.up_proj": 64,
+    "model.layers.10.self_attn.k_proj": 67,
+    "model.layers.10.self_attn.o_proj": 60,
+    "model.layers.10.self_attn.q_proj": 68,
+    "model.layers.10.self_attn.v_proj": 56,
+    "model.layers.11.mlp.down_proj": 63,
+    "model.layers.11.mlp.gate_proj": 64,
+    "model.layers.11.mlp.up_proj": 64,
+    "model.layers.11.self_attn.k_proj": 76,
+    "model.layers.11.self_attn.o_proj": 59,
+    "model.layers.11.self_attn.q_proj": 74,
+    "model.layers.11.self_attn.v_proj": 52,
+    "model.layers.12.mlp.down_proj": 63,
+    "model.layers.12.mlp.gate_proj": 64,
+    "model.layers.12.mlp.up_proj": 64,
+    "model.layers.12.self_attn.k_proj": 81,
+    "model.layers.12.self_attn.o_proj": 58,
+    "model.layers.12.self_attn.q_proj": 77,
+    "model.layers.12.self_attn.v_proj": 53,
+    "model.layers.13.mlp.down_proj": 63,
+    "model.layers.13.mlp.gate_proj": 63,
+    "model.layers.13.mlp.up_proj": 64,
+    "model.layers.13.self_attn.k_proj": 74,
+    "model.layers.13.self_attn.o_proj": 62,
+    "model.layers.13.self_attn.q_proj": 75,
+    "model.layers.13.self_attn.v_proj": 51,
+    "model.layers.14.mlp.down_proj": 63,
+    "model.layers.14.mlp.gate_proj": 63,
+    "model.layers.14.mlp.up_proj": 64,
+    "model.layers.14.self_attn.k_proj": 76,
+    "model.layers.14.self_attn.o_proj": 61,
+    "model.layers.14.self_attn.q_proj": 80,
+    "model.layers.14.self_attn.v_proj": 51,
+    "model.layers.15.mlp.down_proj": 63,
+    "model.layers.15.mlp.gate_proj": 63,
+    "model.layers.15.mlp.up_proj": 64,
+    "model.layers.15.self_attn.k_proj": 68,
+    "model.layers.15.self_attn.o_proj": 64,
+    "model.layers.15.self_attn.q_proj": 70,
+    "model.layers.15.self_attn.v_proj": 56,
+    "model.layers.16.mlp.down_proj": 63,
+    "model.layers.16.mlp.gate_proj": 63,
+    "model.layers.16.mlp.up_proj": 64,
+    "model.layers.16.self_attn.k_proj": 73,
+    "model.layers.16.self_attn.o_proj": 63,
+    "model.layers.16.self_attn.q_proj": 71,
+    "model.layers.16.self_attn.v_proj": 55,
+    "model.layers.17.mlp.down_proj": 64,
+    "model.layers.17.mlp.gate_proj": 63,
+    "model.layers.17.mlp.up_proj": 64,
+    "model.layers.17.self_attn.k_proj": 79,
+    "model.layers.17.self_attn.o_proj": 63,
+    "model.layers.17.self_attn.q_proj": 79,
+    "model.layers.17.self_attn.v_proj": 53,
+    "model.layers.18.mlp.down_proj": 64,
+    "model.layers.18.mlp.gate_proj": 63,
+    "model.layers.18.mlp.up_proj": 64,
+    "model.layers.18.self_attn.k_proj": 71,
+    "model.layers.18.self_attn.o_proj": 63,
+    "model.layers.18.self_attn.q_proj": 72,
+    "model.layers.18.self_attn.v_proj": 58,
+    "model.layers.19.mlp.down_proj": 64,
+    "model.layers.19.mlp.gate_proj": 63,
+    "model.layers.19.mlp.up_proj": 64,
+    "model.layers.19.self_attn.k_proj": 73,
+    "model.layers.19.self_attn.o_proj": 68,
+    "model.layers.19.self_attn.q_proj": 74,
+    "model.layers.19.self_attn.v_proj": 61,
+    "model.layers.2.mlp.down_proj": 61,
+    "model.layers.2.mlp.gate_proj": 51,
+    "model.layers.2.mlp.up_proj": 54,
+    "model.layers.2.self_attn.k_proj": 81,
+    "model.layers.2.self_attn.o_proj": 50,
+    "model.layers.2.self_attn.q_proj": 58,
+    "model.layers.2.self_attn.v_proj": 39,
+    "model.layers.20.mlp.down_proj": 64,
+    "model.layers.20.mlp.gate_proj": 63,
+    "model.layers.20.mlp.up_proj": 64,
+    "model.layers.20.self_attn.k_proj": 74,
+    "model.layers.20.self_attn.o_proj": 63,
+    "model.layers.20.self_attn.q_proj": 73,
+    "model.layers.20.self_attn.v_proj": 57,
+    "model.layers.21.mlp.down_proj": 64,
+    "model.layers.21.mlp.gate_proj": 63,
+    "model.layers.21.mlp.up_proj": 65,
+    "model.layers.21.self_attn.k_proj": 66,
+    "model.layers.21.self_attn.o_proj": 63,
+    "model.layers.21.self_attn.q_proj": 65,
+    "model.layers.21.self_attn.v_proj": 56,
+    "model.layers.22.mlp.down_proj": 64,
+    "model.layers.22.mlp.gate_proj": 64,
+    "model.layers.22.mlp.up_proj": 65,
+    "model.layers.22.self_attn.k_proj": 69,
+    "model.layers.22.self_attn.o_proj": 67,
+    "model.layers.22.self_attn.q_proj": 64,
+    "model.layers.22.self_attn.v_proj": 62,
+    "model.layers.23.mlp.down_proj": 63,
+    "model.layers.23.mlp.gate_proj": 64,
+    "model.layers.23.mlp.up_proj": 65,
+    "model.layers.23.self_attn.k_proj": 74,
+    "model.layers.23.self_attn.o_proj": 65,
+    "model.layers.23.self_attn.q_proj": 71,
+    "model.layers.23.self_attn.v_proj": 61,
+    "model.layers.24.mlp.down_proj": 63,
+    "model.layers.24.mlp.gate_proj": 63,
+    "model.layers.24.mlp.up_proj": 65,
+    "model.layers.24.self_attn.k_proj": 75,
+    "model.layers.24.self_attn.o_proj": 60,
+    "model.layers.24.self_attn.q_proj": 70,
+    "model.layers.24.self_attn.v_proj": 56,
+    "model.layers.25.mlp.down_proj": 64,
+    "model.layers.25.mlp.gate_proj": 62,
+    "model.layers.25.mlp.up_proj": 64,
+    "model.layers.25.self_attn.k_proj": 78,
+    "model.layers.25.self_attn.o_proj": 63,
+    "model.layers.25.self_attn.q_proj": 73,
+    "model.layers.25.self_attn.v_proj": 54,
+    "model.layers.26.mlp.down_proj": 64,
+    "model.layers.26.mlp.gate_proj": 63,
+    "model.layers.26.mlp.up_proj": 65,
+    "model.layers.26.self_attn.k_proj": 76,
+    "model.layers.26.self_attn.o_proj": 66,
+    "model.layers.26.self_attn.q_proj": 75,
+    "model.layers.26.self_attn.v_proj": 58,
+    "model.layers.27.mlp.down_proj": 65,
+    "model.layers.27.mlp.gate_proj": 63,
+    "model.layers.27.mlp.up_proj": 65,
+    "model.layers.27.self_attn.k_proj": 68,
+    "model.layers.27.self_attn.o_proj": 64,
+    "model.layers.27.self_attn.q_proj": 62,
+    "model.layers.27.self_attn.v_proj": 62,
+    "model.layers.28.mlp.down_proj": 65,
+    "model.layers.28.mlp.gate_proj": 63,
+    "model.layers.28.mlp.up_proj": 65,
+    "model.layers.28.self_attn.k_proj": 71,
+    "model.layers.28.self_attn.o_proj": 67,
+    "model.layers.28.self_attn.q_proj": 68,
+    "model.layers.28.self_attn.v_proj": 64,
+    "model.layers.29.mlp.down_proj": 65,
+    "model.layers.29.mlp.gate_proj": 63,
+    "model.layers.29.mlp.up_proj": 65,
+    "model.layers.29.self_attn.k_proj": 68,
+    "model.layers.29.self_attn.o_proj": 67,
+    "model.layers.29.self_attn.q_proj": 66,
+    "model.layers.29.self_attn.v_proj": 63,
+    "model.layers.3.mlp.down_proj": 62,
+    "model.layers.3.mlp.gate_proj": 56,
+    "model.layers.3.mlp.up_proj": 58,
+    "model.layers.3.self_attn.k_proj": 83,
+    "model.layers.3.self_attn.o_proj": 59,
+    "model.layers.3.self_attn.q_proj": 70,
+    "model.layers.3.self_attn.v_proj": 46,
+    "model.layers.30.mlp.down_proj": 64,
+    "model.layers.30.mlp.gate_proj": 64,
+    "model.layers.30.mlp.up_proj": 65,
+    "model.layers.30.self_attn.k_proj": 54,
+    "model.layers.30.self_attn.o_proj": 67,
+    "model.layers.30.self_attn.q_proj": 52,
+    "model.layers.30.self_attn.v_proj": 65,
+    "model.layers.31.mlp.down_proj": 64,
+    "model.layers.31.mlp.gate_proj": 64,
+    "model.layers.31.mlp.up_proj": 65,
+    "model.layers.31.self_attn.k_proj": 68,
+    "model.layers.31.self_attn.o_proj": 65,
+    "model.layers.31.self_attn.q_proj": 67,
+    "model.layers.31.self_attn.v_proj": 62,
+    "model.layers.32.mlp.down_proj": 64,
+    "model.layers.32.mlp.gate_proj": 64,
+    "model.layers.32.mlp.up_proj": 65,
+    "model.layers.32.self_attn.k_proj": 72,
+    "model.layers.32.self_attn.o_proj": 63,
+    "model.layers.32.self_attn.q_proj": 71,
+    "model.layers.32.self_attn.v_proj": 62,
+    "model.layers.33.mlp.down_proj": 64,
+    "model.layers.33.mlp.gate_proj": 63,
+    "model.layers.33.mlp.up_proj": 65,
+    "model.layers.33.self_attn.k_proj": 75,
+    "model.layers.33.self_attn.o_proj": 63,
+    "model.layers.33.self_attn.q_proj": 77,
+    "model.layers.33.self_attn.v_proj": 60,
+    "model.layers.34.mlp.down_proj": 64,
+    "model.layers.34.mlp.gate_proj": 63,
+    "model.layers.34.mlp.up_proj": 65,
+    "model.layers.34.self_attn.k_proj": 68,
+    "model.layers.34.self_attn.o_proj": 62,
+    "model.layers.34.self_attn.q_proj": 64,
+    "model.layers.34.self_attn.v_proj": 61,
+    "model.layers.35.mlp.down_proj": 64,
+    "model.layers.35.mlp.gate_proj": 63,
+    "model.layers.35.mlp.up_proj": 65,
+    "model.layers.35.self_attn.k_proj": 66,
+    "model.layers.35.self_attn.o_proj": 61,
+    "model.layers.35.self_attn.q_proj": 67,
+    "model.layers.35.self_attn.v_proj": 60,
+    "model.layers.36.mlp.down_proj": 65,
+    "model.layers.36.mlp.gate_proj": 63,
+    "model.layers.36.mlp.up_proj": 65,
+    "model.layers.36.self_attn.k_proj": 69,
+    "model.layers.36.self_attn.o_proj": 60,
+    "model.layers.36.self_attn.q_proj": 69,
+    "model.layers.36.self_attn.v_proj": 61,
+    "model.layers.37.mlp.down_proj": 65,
+    "model.layers.37.mlp.gate_proj": 63,
+    "model.layers.37.mlp.up_proj": 65,
+    "model.layers.37.self_attn.k_proj": 73,
+    "model.layers.37.self_attn.o_proj": 62,
+    "model.layers.37.self_attn.q_proj": 75,
+    "model.layers.37.self_attn.v_proj": 61,
+    "model.layers.38.mlp.down_proj": 65,
+    "model.layers.38.mlp.gate_proj": 63,
+    "model.layers.38.mlp.up_proj": 65,
+    "model.layers.38.self_attn.k_proj": 72,
+    "model.layers.38.self_attn.o_proj": 60,
+    "model.layers.38.self_attn.q_proj": 76,
+    "model.layers.38.self_attn.v_proj": 59,
+    "model.layers.39.mlp.down_proj": 65,
+    "model.layers.39.mlp.gate_proj": 63,
+    "model.layers.39.mlp.up_proj": 65,
+    "model.layers.39.self_attn.k_proj": 77,
+    "model.layers.39.self_attn.o_proj": 61,
+    "model.layers.39.self_attn.q_proj": 80,
+    "model.layers.39.self_attn.v_proj": 61,
+    "model.layers.4.mlp.down_proj": 62,
+    "model.layers.4.mlp.gate_proj": 59,
+    "model.layers.4.mlp.up_proj": 60,
+    "model.layers.4.self_attn.k_proj": 74,
+    "model.layers.4.self_attn.o_proj": 55,
+    "model.layers.4.self_attn.q_proj": 71,
+    "model.layers.4.self_attn.v_proj": 44,
+    "model.layers.40.mlp.down_proj": 65,
+    "model.layers.40.mlp.gate_proj": 63,
+    "model.layers.40.mlp.up_proj": 65,
+    "model.layers.40.self_attn.k_proj": 73,
+    "model.layers.40.self_attn.o_proj": 68,
+    "model.layers.40.self_attn.q_proj": 79,
+    "model.layers.40.self_attn.v_proj": 65,
+    "model.layers.41.mlp.down_proj": 66,
+    "model.layers.41.mlp.gate_proj": 63,
+    "model.layers.41.mlp.up_proj": 64,
+    "model.layers.41.self_attn.k_proj": 70,
+    "model.layers.41.self_attn.o_proj": 65,
+    "model.layers.41.self_attn.q_proj": 74,
+    "model.layers.41.self_attn.v_proj": 63,
+    "model.layers.42.mlp.down_proj": 65,
+    "model.layers.42.mlp.gate_proj": 64,
+    "model.layers.42.mlp.up_proj": 64,
+    "model.layers.42.self_attn.k_proj": 68,
+    "model.layers.42.self_attn.o_proj": 65,
+    "model.layers.42.self_attn.q_proj": 74,
+    "model.layers.42.self_attn.v_proj": 65,
+    "model.layers.43.mlp.down_proj": 66,
+    "model.layers.43.mlp.gate_proj": 64,
+    "model.layers.43.mlp.up_proj": 64,
+    "model.layers.43.self_attn.k_proj": 69,
+    "model.layers.43.self_attn.o_proj": 65,
+    "model.layers.43.self_attn.q_proj": 70,
+    "model.layers.43.self_attn.v_proj": 64,
+    "model.layers.44.mlp.down_proj": 65,
+    "model.layers.44.mlp.gate_proj": 65,
+    "model.layers.44.mlp.up_proj": 63,
+    "model.layers.44.self_attn.k_proj": 67,
+    "model.layers.44.self_attn.o_proj": 72,
+    "model.layers.44.self_attn.q_proj": 76,
+    "model.layers.44.self_attn.v_proj": 67,
+    "model.layers.45.mlp.down_proj": 65,
+    "model.layers.45.mlp.gate_proj": 65,
+    "model.layers.45.mlp.up_proj": 63,
+    "model.layers.45.self_attn.k_proj": 64,
+    "model.layers.45.self_attn.o_proj": 70,
+    "model.layers.45.self_attn.q_proj": 66,
+    "model.layers.45.self_attn.v_proj": 70,
+    "model.layers.46.mlp.down_proj": 65,
+    "model.layers.46.mlp.gate_proj": 66,
+    "model.layers.46.mlp.up_proj": 63,
+    "model.layers.46.self_attn.k_proj": 61,
+    "model.layers.46.self_attn.o_proj": 71,
+    "model.layers.46.self_attn.q_proj": 63,
+    "model.layers.46.self_attn.v_proj": 69,
+    "model.layers.47.mlp.down_proj": 65,
+    "model.layers.47.mlp.gate_proj": 66,
+    "model.layers.47.mlp.up_proj": 63,
+    "model.layers.47.self_attn.k_proj": 71,
+    "model.layers.47.self_attn.o_proj": 68,
+    "model.layers.47.self_attn.q_proj": 73,
+    "model.layers.47.self_attn.v_proj": 67,
+    "model.layers.48.mlp.down_proj": 65,
+    "model.layers.48.mlp.gate_proj": 67,
+    "model.layers.48.mlp.up_proj": 63,
+    "model.layers.48.self_attn.k_proj": 54,
+    "model.layers.48.self_attn.o_proj": 71,
+    "model.layers.48.self_attn.q_proj": 58,
+    "model.layers.48.self_attn.v_proj": 71,
+    "model.layers.49.mlp.down_proj": 65,
+    "model.layers.49.mlp.gate_proj": 67,
+    "model.layers.49.mlp.up_proj": 64,
+    "model.layers.49.self_attn.k_proj": 53,
+    "model.layers.49.self_attn.o_proj": 67,
+    "model.layers.49.self_attn.q_proj": 50,
+    "model.layers.49.self_attn.v_proj": 70,
+    "model.layers.5.mlp.down_proj": 62,
+    "model.layers.5.mlp.gate_proj": 60,
+    "model.layers.5.mlp.up_proj": 61,
+    "model.layers.5.self_attn.k_proj": 80,
+    "model.layers.5.self_attn.o_proj": 57,
+    "model.layers.5.self_attn.q_proj": 77,
+    "model.layers.5.self_attn.v_proj": 46,
+    "model.layers.50.mlp.down_proj": 65,
+    "model.layers.50.mlp.gate_proj": 67,
+    "model.layers.50.mlp.up_proj": 64,
+    "model.layers.50.self_attn.k_proj": 50,
+    "model.layers.50.self_attn.o_proj": 70,
+    "model.layers.50.self_attn.q_proj": 53,
+    "model.layers.50.self_attn.v_proj": 71,
+    "model.layers.51.mlp.down_proj": 65,
+    "model.layers.51.mlp.gate_proj": 67,
+    "model.layers.51.mlp.up_proj": 64,
+    "model.layers.51.self_attn.k_proj": 56,
+    "model.layers.51.self_attn.o_proj": 68,
+    "model.layers.51.self_attn.q_proj": 59,
+    "model.layers.51.self_attn.v_proj": 72,
+    "model.layers.52.mlp.down_proj": 65,
+    "model.layers.52.mlp.gate_proj": 67,
+    "model.layers.52.mlp.up_proj": 64,
+    "model.layers.52.self_attn.k_proj": 57,
+    "model.layers.52.self_attn.o_proj": 70,
+    "model.layers.52.self_attn.q_proj": 66,
+    "model.layers.52.self_attn.v_proj": 69,
+    "model.layers.53.mlp.down_proj": 65,
+    "model.layers.53.mlp.gate_proj": 67,
+    "model.layers.53.mlp.up_proj": 64,
+    "model.layers.53.self_attn.k_proj": 46,
+    "model.layers.53.self_attn.o_proj": 70,
+    "model.layers.53.self_attn.q_proj": 49,
+    "model.layers.53.self_attn.v_proj": 74,
+    "model.layers.54.mlp.down_proj": 65,
+    "model.layers.54.mlp.gate_proj": 68,
+    "model.layers.54.mlp.up_proj": 64,
+    "model.layers.54.self_attn.k_proj": 46,
+    "model.layers.54.self_attn.o_proj": 69,
+    "model.layers.54.self_attn.q_proj": 50,
+    "model.layers.54.self_attn.v_proj": 73,
+    "model.layers.55.mlp.down_proj": 65,
+    "model.layers.55.mlp.gate_proj": 68,
+    "model.layers.55.mlp.up_proj": 65,
+    "model.layers.55.self_attn.k_proj": 56,
+    "model.layers.55.self_attn.o_proj": 72,
+    "model.layers.55.self_attn.q_proj": 59,
+    "model.layers.55.self_attn.v_proj": 77,
+    "model.layers.56.mlp.down_proj": 65,
+    "model.layers.56.mlp.gate_proj": 68,
+    "model.layers.56.mlp.up_proj": 65,
+    "model.layers.56.self_attn.k_proj": 55,
+    "model.layers.56.self_attn.o_proj": 68,
+    "model.layers.56.self_attn.q_proj": 56,
+    "model.layers.56.self_attn.v_proj": 73,
+    "model.layers.57.mlp.down_proj": 65,
+    "model.layers.57.mlp.gate_proj": 68,
+    "model.layers.57.mlp.up_proj": 65,
+    "model.layers.57.self_attn.k_proj": 48,
+    "model.layers.57.self_attn.o_proj": 72,
+    "model.layers.57.self_attn.q_proj": 52,
+    "model.layers.57.self_attn.v_proj": 77,
+    "model.layers.58.mlp.down_proj": 65,
+    "model.layers.58.mlp.gate_proj": 68,
+    "model.layers.58.mlp.up_proj": 65,
+    "model.layers.58.self_attn.k_proj": 44,
+    "model.layers.58.self_attn.o_proj": 69,
+    "model.layers.58.self_attn.q_proj": 46,
+    "model.layers.58.self_attn.v_proj": 73,
+    "model.layers.59.mlp.down_proj": 65,
+    "model.layers.59.mlp.gate_proj": 68,
+    "model.layers.59.mlp.up_proj": 65,
+    "model.layers.59.self_attn.k_proj": 45,
+    "model.layers.59.self_attn.o_proj": 68,
+    "model.layers.59.self_attn.q_proj": 47,
+    "model.layers.59.self_attn.v_proj": 74,
+    "model.layers.6.mlp.down_proj": 62,
+    "model.layers.6.mlp.gate_proj": 60,
+    "model.layers.6.mlp.up_proj": 61,
+    "model.layers.6.self_attn.k_proj": 80,
+    "model.layers.6.self_attn.o_proj": 61,
+    "model.layers.6.self_attn.q_proj": 80,
+    "model.layers.6.self_attn.v_proj": 47,
+    "model.layers.60.mlp.down_proj": 65,
+    "model.layers.60.mlp.gate_proj": 68,
+    "model.layers.60.mlp.up_proj": 66,
+    "model.layers.60.self_attn.k_proj": 27,
+    "model.layers.60.self_attn.o_proj": 67,
+    "model.layers.60.self_attn.q_proj": 28,
+    "model.layers.60.self_attn.v_proj": 74,
+    "model.layers.61.mlp.down_proj": 65,
+    "model.layers.61.mlp.gate_proj": 68,
+    "model.layers.61.mlp.up_proj": 66,
+    "model.layers.61.self_attn.k_proj": 41,
+    "model.layers.61.self_attn.o_proj": 70,
+    "model.layers.61.self_attn.q_proj": 43,
+    "model.layers.61.self_attn.v_proj": 76,
+    "model.layers.62.mlp.down_proj": 65,
+    "model.layers.62.mlp.gate_proj": 68,
+    "model.layers.62.mlp.up_proj": 66,
+    "model.layers.62.self_attn.k_proj": 42,
+    "model.layers.62.self_attn.o_proj": 68,
+    "model.layers.62.self_attn.q_proj": 42,
+    "model.layers.62.self_attn.v_proj": 75,
+    "model.layers.63.mlp.down_proj": 65,
+    "model.layers.63.mlp.gate_proj": 68,
+    "model.layers.63.mlp.up_proj": 66,
+    "model.layers.63.self_attn.k_proj": 40,
+    "model.layers.63.self_attn.o_proj": 65,
+    "model.layers.63.self_attn.q_proj": 43,
+    "model.layers.63.self_attn.v_proj": 72,
+    "model.layers.64.mlp.down_proj": 65,
+    "model.layers.64.mlp.gate_proj": 68,
+    "model.layers.64.mlp.up_proj": 66,
+    "model.layers.64.self_attn.k_proj": 50,
+    "model.layers.64.self_attn.o_proj": 71,
+    "model.layers.64.self_attn.q_proj": 51,
+    "model.layers.64.self_attn.v_proj": 77,
+    "model.layers.65.mlp.down_proj": 65,
+    "model.layers.65.mlp.gate_proj": 68,
+    "model.layers.65.mlp.up_proj": 66,
+    "model.layers.65.self_attn.k_proj": 33,
+    "model.layers.65.self_attn.o_proj": 64,
+    "model.layers.65.self_attn.q_proj": 35,
+    "model.layers.65.self_attn.v_proj": 71,
+    "model.layers.66.mlp.down_proj": 65,
+    "model.layers.66.mlp.gate_proj": 68,
+    "model.layers.66.mlp.up_proj": 67,
+    "model.layers.66.self_attn.k_proj": 41,
+    "model.layers.66.self_attn.o_proj": 68,
+    "model.layers.66.self_attn.q_proj": 43,
+    "model.layers.66.self_attn.v_proj": 74,
+    "model.layers.67.mlp.down_proj": 65,
+    "model.layers.67.mlp.gate_proj": 68,
+    "model.layers.67.mlp.up_proj": 67,
+    "model.layers.67.self_attn.k_proj": 35,
+    "model.layers.67.self_attn.o_proj": 57,
+    "model.layers.67.self_attn.q_proj": 34,
+    "model.layers.67.self_attn.v_proj": 67,
+    "model.layers.68.mlp.down_proj": 65,
+    "model.layers.68.mlp.gate_proj": 68,
+    "model.layers.68.mlp.up_proj": 67,
+    "model.layers.68.self_attn.k_proj": 54,
+    "model.layers.68.self_attn.o_proj": 71,
+    "model.layers.68.self_attn.q_proj": 56,
+    "model.layers.68.self_attn.v_proj": 80,
+    "model.layers.69.mlp.down_proj": 65,
+    "model.layers.69.mlp.gate_proj": 68,
+    "model.layers.69.mlp.up_proj": 67,
+    "model.layers.69.self_attn.k_proj": 65,
+    "model.layers.69.self_attn.o_proj": 72,
+    "model.layers.69.self_attn.q_proj": 65,
+    "model.layers.69.self_attn.v_proj": 84,
+    "model.layers.7.mlp.down_proj": 63,
+    "model.layers.7.mlp.gate_proj": 61,
+    "model.layers.7.mlp.up_proj": 62,
+    "model.layers.7.self_attn.k_proj": 81,
+    "model.layers.7.self_attn.o_proj": 59,
+    "model.layers.7.self_attn.q_proj": 82,
+    "model.layers.7.self_attn.v_proj": 45,
+    "model.layers.70.mlp.down_proj": 65,
+    "model.layers.70.mlp.gate_proj": 68,
+    "model.layers.70.mlp.up_proj": 68,
+    "model.layers.70.self_attn.k_proj": 58,
+    "model.layers.70.self_attn.o_proj": 66,
+    "model.layers.70.self_attn.q_proj": 60,
+    "model.layers.70.self_attn.v_proj": 77,
+    "model.layers.71.mlp.down_proj": 65,
+    "model.layers.71.mlp.gate_proj": 68,
+    "model.layers.71.mlp.up_proj": 68,
+    "model.layers.71.self_attn.k_proj": 65,
+    "model.layers.71.self_attn.o_proj": 68,
+    "model.layers.71.self_attn.q_proj": 65,
+    "model.layers.71.self_attn.v_proj": 77,
+    "model.layers.72.mlp.down_proj": 65,
+    "model.layers.72.mlp.gate_proj": 67,
+    "model.layers.72.mlp.up_proj": 68,
+    "model.layers.72.self_attn.k_proj": 70,
+    "model.layers.72.self_attn.o_proj": 69,
+    "model.layers.72.self_attn.q_proj": 72,
+    "model.layers.72.self_attn.v_proj": 77,
+    "model.layers.73.mlp.down_proj": 65,
+    "model.layers.73.mlp.gate_proj": 67,
+    "model.layers.73.mlp.up_proj": 68,
+    "model.layers.73.self_attn.k_proj": 67,
+    "model.layers.73.self_attn.o_proj": 69,
+    "model.layers.73.self_attn.q_proj": 68,
+    "model.layers.73.self_attn.v_proj": 82,
+    "model.layers.74.mlp.down_proj": 65,
+    "model.layers.74.mlp.gate_proj": 67,
+    "model.layers.74.mlp.up_proj": 68,
+    "model.layers.74.self_attn.k_proj": 68,
+    "model.layers.74.self_attn.o_proj": 65,
+    "model.layers.74.self_attn.q_proj": 70,
+    "model.layers.74.self_attn.v_proj": 77,
+    "model.layers.75.mlp.down_proj": 65,
+    "model.layers.75.mlp.gate_proj": 67,
+    "model.layers.75.mlp.up_proj": 68,
+    "model.layers.75.self_attn.k_proj": 64,
+    "model.layers.75.self_attn.o_proj": 65,
+    "model.layers.75.self_attn.q_proj": 68,
+    "model.layers.75.self_attn.v_proj": 82,
+    "model.layers.76.mlp.down_proj": 65,
+    "model.layers.76.mlp.gate_proj": 67,
+    "model.layers.76.mlp.up_proj": 68,
+    "model.layers.76.self_attn.k_proj": 67,
+    "model.layers.76.self_attn.o_proj": 71,
+    "model.layers.76.self_attn.q_proj": 75,
+    "model.layers.76.self_attn.v_proj": 89,
+    "model.layers.77.mlp.down_proj": 65,
+    "model.layers.77.mlp.gate_proj": 67,
+    "model.layers.77.mlp.up_proj": 68,
+    "model.layers.77.self_attn.k_proj": 64,
+    "model.layers.77.self_attn.o_proj": 65,
+    "model.layers.77.self_attn.q_proj": 69,
+    "model.layers.77.self_attn.v_proj": 84,
+    "model.layers.78.mlp.down_proj": 64,
+    "model.layers.78.mlp.gate_proj": 67,
+    "model.layers.78.mlp.up_proj": 68,
+    "model.layers.78.self_attn.k_proj": 64,
+    "model.layers.78.self_attn.o_proj": 60,
+    "model.layers.78.self_attn.q_proj": 66,
+    "model.layers.78.self_attn.v_proj": 77,
+    "model.layers.79.mlp.down_proj": 63,
+    "model.layers.79.mlp.gate_proj": 68,
+    "model.layers.79.mlp.up_proj": 67,
+    "model.layers.79.self_attn.k_proj": 65,
+    "model.layers.79.self_attn.o_proj": 48,
+    "model.layers.79.self_attn.q_proj": 61,
+    "model.layers.79.self_attn.v_proj": 62,
+    "model.layers.8.mlp.down_proj": 64,
+    "model.layers.8.mlp.gate_proj": 62,
+    "model.layers.8.mlp.up_proj": 62,
+    "model.layers.8.self_attn.k_proj": 82,
+    "model.layers.8.self_attn.o_proj": 61,
+    "model.layers.8.self_attn.q_proj": 82,
+    "model.layers.8.self_attn.v_proj": 46,
+    "model.layers.9.mlp.down_proj": 64,
+    "model.layers.9.mlp.gate_proj": 63,
+    "model.layers.9.mlp.up_proj": 63,
+    "model.layers.9.self_attn.k_proj": 77,
+    "model.layers.9.self_attn.o_proj": 62,
+    "model.layers.9.self_attn.q_proj": 76,
+    "model.layers.9.self_attn.v_proj": 51
+  },
+  "rank_file": "/workspace/asr/peft/qlora/llama2_70bhf_lazylora_r64_2.json",
+  "target_modules": [
+    "o_proj",
+    "up_proj",
+    "k_proj",
+    "down_proj",
+    "q_proj",
+    "v_proj",
+    "gate_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "token_dim": 8192
+}

adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:857b3b46b329f38aadcc7ddf874025e17ec34f657ce778cd4a8e4a589455bbce
+size 1657058821

usage.py

@@ -0,0 +1,51 @@
+import sys
+sys.path.insert(1, '/workspace/asr/peft/src')
+# TODO set this path to the lazy-lora source code path, or you can install it from source code:
+# TODO, please install lazylora for usage:
+# git clone [email protected]:Xianchao-Wu/peft.git
+# cd peft
+# python setup.py install
+
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
+from peft import PeftModel, PeftConfig
+import os
+import torch
+
+#import ipdb; ipdb.set_trace()
+cache_dir="/workspace/asr/peft/qlora" 
+# TODO set this cache_dir to the path where you stored (or, want to store) llama2-70b-hf model
+
+lazylora_dir=os.getcwd() # the path that contains 'adapter_config.json' and 'adapter_model.bin'
+
+config = PeftConfig.from_pretrained(lazylora_dir)
+
+tokenizer = AutoTokenizer.from_pretrained(
+    config.base_model_name_or_path, 
+    cache_dir=cache_dir,
+    use_auth_token=True
+)
+
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type='nf4',
+    bnb_4bit_compute_dtype=torch.bfloat16
+)
+
+model = AutoModelForCausalLM.from_pretrained(
+    config.base_model_name_or_path,
+    quantization_config=bnb_config,
+    device_map="auto",
+    cache_dir=cache_dir,
+    use_auth_token=True
+)
+#model.print_trainable_parameters()
+print(sum(p.numel() for p in model.parameters()))
+# 34,751,127,552 -> half-size of 70B due to 4-bit loading
+
+model = PeftModel.from_pretrained(model, lazylora_dir)
+print('after adding lazy lora parameters:')
+model.print_trainable_parameters()
+# trainable params: 0 || all params: 35,579,442,176 || trainable%: 0.0
+
+