Edit model card

This model is randomly initialized, using the config from Qwen/Qwen2-72B-Instruct but with smaller size.

Codes:

import transformers
import torch
import os
from huggingface_hub import create_repo, upload_folder
import accelerate

source_model_id = 'Qwen/Qwen2-72B-Instruct'
save_path = '/tmp/yujiepan/qwen2-tiny-random'
repo_id = 'yujiepan/qwen2-tiny-random'

os.system(f'rm -rf {save_path}')

config = transformers.AutoConfig.from_pretrained(
    source_model_id,
    trust_remote_code=True,
)
config._name_or_path = source_model_id
config.hidden_size = 8
config.intermediate_size = 16
config.num_key_value_heads = 2
config.num_attention_heads = 4
config.num_hidden_layers = 2
config.max_window_layers = 1

model = transformers.AutoModelForCausalLM.from_config(
    config,
    trust_remote_code=True,
)
model.generation_config = transformers.GenerationConfig.from_pretrained(source_model_id)
model = model.to(torch.bfloat16)

with torch.no_grad():
    for p in model.parameters():
        torch.nn.init.normal_(p)

model.save_pretrained(save_path)

tokenizer = transformers.AutoTokenizer.from_pretrained(
    source_model_id,
    trust_remote_code=True,
)
tokenizer.save_pretrained(save_path)

output = model.float().generate(torch.tensor([[1, 2, 3]]).long(), max_length=16, do_sample=True)

os.system(f'ls -alh {save_path}')
# os.system(f'rm -rf {save_path}/model.safetensors')
create_repo(repo_id, exist_ok=True)
upload_folder(repo_id=repo_id, folder_path=save_path)
Downloads last month
2
Safetensors
Model size
2.43M params
Tensor type
BF16
·
Inference Examples
This model does not have enough activity to be deployed to Inference API (serverless) yet. Increase its social visibility and check back later, or deploy to Inference Endpoints (dedicated) instead.

Collection including yujiepan/qwen2-tiny-random