Upload 8 files

app.py

@@ -0,0 +1,135 @@
+import fn
+import gradio as gr
+import models
+
+def fn_chat(instruction, input, model, dtype, is_messages, template, max_new_tokens, temperature, top_p, top_k, repetition_penalty):
+    args = {
+        'instruction': instruction,
+        'input': input,
+        'model': model,
+        'dtype': dtype,
+        'is_messages': is_messages,
+        'template': template,
+        'max_new_tokens': int(max_new_tokens),
+        'temperature': float(temperature),
+        'top_p': float(top_p),
+        'top_k': int(top_k),
+        'repetition_penalty': float(repetition_penalty),
+    }
+
+    content = fn.infer(args)
+    return content
+
+with gr.Blocks() as demo:
+    opt = models.get_head_options()
+
+    with gr.Row():
+        with gr.Column(scale=1):
+            model = gr.Textbox(
+                value=opt['model'],
+                label='model',
+                show_label=True,
+                interactive=True,
+                show_copy_button=True,
+            )
+
+            dtype = gr.Dropdown(
+                value=opt['dtype'],
+                choices=['int4','int8','fp16', 'bf16'],
+                label='dtype',
+                show_label=True,
+                interactive=True,
+                allow_custom_value=True,
+            )
+            template = gr.Textbox(
+                value=opt['template'],
+                lines=3,
+                label='template',
+                show_label=True,
+                interactive=True,
+                show_copy_button=True,
+                )
+            is_messages = gr.Checkbox(
+                value=opt['is_messages'],
+                label='is_messages',
+                show_label=True,
+                interactive=True,
+                )
+
+        with gr.Column(scale=1):
+            max_new_tokens = gr.Textbox(
+                value=opt['max_new_tokens'],
+                label='max_new_tokens',
+                show_label=True,
+                interactive=True,
+                show_copy_button=True,
+                )
+            temperature = gr.Textbox(
+                value=opt['temperature'],
+                label='temperature',
+                show_label=True,
+                interactive=True,
+                show_copy_button=True,
+                )
+            top_p = gr.Textbox(
+                value=opt['top_p'],
+                label='top_p',
+                show_label=True,
+                interactive=True,
+                show_copy_button=True,
+                )
+            top_k = gr.Textbox(
+                value=opt['top_k'],
+                label='top_k',
+                show_label=True,
+                interactive=True,
+                show_copy_button=True,
+                )
+            repetition_penalty = gr.Textbox(
+                value=opt['repetition_penalty'],
+                label='repetition_penalty',
+                show_label=True,
+                interactive=True,
+                show_copy_button=True,
+                )
+
+    with gr.Accordion('Preset', open=False):
+        gr.Examples(
+            models.get_examples(),
+            [model, dtype, is_messages, template, max_new_tokens, temperature, top_p, top_k, repetition_penalty],
+        )
+
+    with gr.Row():
+        with gr.Column(scale=1):
+            instruction = gr.Textbox(
+                lines=20,
+                label='instruction',
+                show_label=True,
+                interactive=True,
+                show_copy_button=True,
+                )
+            user_input = gr.Textbox(
+                lines=1,
+                label='input',
+                show_label=True,
+                interactive=True,
+                show_copy_button=True,
+                )
+            chat_button = gr.Button(value='chat')
+
+        with gr.Column(scale=1):
+            said = gr.Textbox(
+                label='said',
+                lines=15,
+                show_label=True,
+                show_copy_button=True,
+                )
+
+    chat_button.click(
+        fn=fn_chat,
+        inputs=[instruction, user_input, model, dtype, is_messages, template, max_new_tokens, temperature, top_p, top_k, repetition_penalty],
+        outputs=[said],
+        )
+
+if __name__ == '__main__':
+    demo.launch()

fn.py

@@ -0,0 +1,128 @@
+import os
+import re
+import torch
+import datetime
+import json
+import csv
+import gc
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
+from transformers import TextStreamer, TextIteratorStreamer
+from transformers import GenerationConfig, AutoConfig, GPTQConfig, AwqConfig
+from models import models
+
+tokenizer = None
+model = None
+loaded_model_name = None
+loaded_dtype = None
+
+def load_model(model_name, dtype = 'int4'):
+    global tokenizer, model, loaded_model_name, loaded_dtype
+
+    if loaded_model_name == model_name and loaded_dtype == dtype:
+        return
+
+    del model
+    del tokenizer
+    model = None
+    tokenizer = None
+    gc.collect()
+    torch.cuda.empty_cache()
+
+    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+    
+    if dtype == 'int4':
+        model = AutoModelForCausalLM.from_pretrained(
+            model_name,
+            device_map="auto",
+            trust_remote_code=True,
+            quantization_config=BitsAndBytesConfig(
+                load_in_4bit=True,
+                bnb_4bit_compute_dtype=torch.bfloat16,
+            ),
+        )
+    elif dtype == 'int8':
+        model = AutoModelForCausalLM.from_pretrained(
+            model_name,
+            device_map="auto",
+            trust_remote_code=True,
+            quantization_config=BitsAndBytesConfig(
+                torch_dtype=torch.bfloat16,
+                load_in_8bit=True,
+            ),
+        )
+    elif dtype == 'fp16':
+        model = AutoModelForCausalLM.from_pretrained(
+            model_name,
+            device_map="auto",
+            trust_remote_code=True,
+            torch_dtype=torch.float16,
+        )
+    elif dtype == 'bf16':
+        model = AutoModelForCausalLM.from_pretrained(
+            model_name,
+            device_map="auto",
+            trust_remote_code=True,
+            torch_dtype=torch.bfloat16,
+        )
+    else:
+        model = AutoModelForCausalLM.from_pretrained(
+            model_name,
+            trust_remote_code=True,
+            device_map="auto",
+        )
+
+    loaded_model_name = model_name
+    loaded_dtype = dtype
+
+def infer(args: dict):
+    global tokenizer, model, loaded_model_name
+    
+    if 'model' in args:
+        args['model_name'] = args['model']
+
+    if not tokenizer or 'model_name' in args and loaded_model_name != args['model_name']:
+        if 'dtype' in args:
+            load_model(args['model_name'], args['dtype'])
+        else:
+            load_model(args['model_name'])
+
+    config = {}
+    if args['model_name'] in models:
+        config = models[args['model_name']]
+    config.update(args)
+
+    if config['is_messages']:
+        messages = []
+        messages.append({"role": "system", "content": args['instruction']})
+        if args['input']:
+            messages.append({"role": "user", "content": args['input']})
+        tprompt = tokenizer.apply_chat_template(conversation=messages, add_generation_prompt=True, tokenize=False)
+    else:
+        tprompt = config['template'].format(bos_token=tokenizer.bos_token, instruction=args['instruction'], input=args['input'])
+
+    kwargs = config.copy()
+    for k in ['model_name', 'template', 'instruction', 'input', 'location', 'endpoint', 'model', 'dtype', 'is_messages']:
+        if k in kwargs:
+            del kwargs[k]
+
+    with torch.no_grad():
+        token_ids = tokenizer.encode(tprompt, add_special_tokens=False, return_tensors="pt")
+        if config['is_messages']:
+            output_ids = model.generate(
+                input_ids=token_ids.to(model.device),
+                do_sample=True,
+                **kwargs,
+            )
+        else:
+            output_ids = model.generate(
+                input_ids=token_ids.to(model.device),
+                do_sample=True,
+                pad_token_id=tokenizer.pad_token_id,
+                bos_token_id=tokenizer.bos_token_id,
+                eos_token_id=tokenizer.eos_token_id,
+                **kwargs,
+            )
+    out = output_ids.tolist()[0][token_ids.size(1) :]
+    content = tokenizer.decode(out, skip_special_tokens=True)
+
+    return content

install.bat

@@ -0,0 +1,56 @@
+@echo off
+
+rem -------------------------------------------
+rem NOT guaranteed to work on Windows
+
+set REPOS=https://huggingface.co/spaces/aka7774/llm
+set APPDIR=llm
+set VENV=venv
+
+rem -------------------------------------------
+
+set INSTALL_DIR=%~dp0
+cd /d %INSTALL_DIR%
+
+:git_clone
+set DL_URL=%REPOS%
+set DL_DST=%APPDIR%
+git clone %DL_URL% %APPDIR%
+if exist %DL_DST% goto install_python
+
+set DL_URL=https://github.com/git-for-windows/git/releases/download/v2.41.0.windows.3/PortableGit-2.41.0.3-64-bit.7z.exe
+set DL_DST=PortableGit-2.41.0.3-64-bit.7z.exe
+curl -L -o %DL_DST% %DL_URL%
+if not exist %DL_DST% bitsadmin /transfer dl %DL_URL% %DL_DST%
+%DL_DST% -y
+del %DL_DST%
+
+set GIT=%INSTALL_DIR%PortableGit\bin\git
+%GIT% clone %REPOS%
+
+:install_python
+set DL_URL=https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.10.13+20240107-i686-pc-windows-msvc-shared-install_only.tar.gz
+set DL_DST="%INSTALL_DIR%python.tar.gz"
+curl -L -o %DL_DST% %DL_URL%
+if not exist %DL_DST% bitsadmin /transfer dl %DL_URL% %DL_DST%
+tar -xzf %DL_DST%
+
+set PYTHON=%INSTALL_DIR%python\python.exe
+set PATH=%PATH%;%INSTALL_DIR%python310\Scripts
+
+:install_venv
+cd %APPDIR%
+%PYTHON% -m venv %VENV%
+set PYTHON=%VENV%\Scripts\python.exe
+
+:install_pip
+set DL_URL=https://bootstrap.pypa.io/get-pip.py
+set DL_DST=%INSTALL_DIR%get-pip.py
+curl -o %DL_DST% %DL_URL%
+if not exist %DL_DST% bitsadmin /transfer dl %DL_URL% %DL_DST%
+%PYTHON% %DL_DST%
+
+%PYTHON% -m pip install gradio
+%PYTHON% -m pip install -r requirements.txt
+
+pause

main.py

@@ -0,0 +1,33 @@
+import os
+import sys
+import time
+import signal
+import io
+
+from fastapi import FastAPI, Request, status, Form, UploadFile
+from fastapi.staticfiles import StaticFiles
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel, Field
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse
+
+import fn
+import gradio as gr
+from app import demo
+
+app = FastAPI()
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=['*'],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+gr.mount_gradio_app(app, demo, path="/gradio")
+
+@app.post("/infer")
+async def api_infer(args: dict):
+    content = fn.infer(args)
+    return {'content': content}

models.json

@@ -0,0 +1,79 @@
+{
+    "lightblue/qarasu-14B-chat-plus-unleashed": {
+		"model": "lightblue/qarasu-14B-chat-plus-unleashed",
+		"dtype": "int4",
+		"is_messages": false,
+		"template": "{instruction}\n\n{input}",
+		"max_new_tokens": 256,
+		"temperature": 0.1,
+		"top_p": 0.9,
+		"top_k": 40,
+		"repetition_penalty": 1.1
+	},
+    "elyza/ELYZA-japanese-Llama-2-13b-fast-instruct": {
+		"model": "elyza/ELYZA-japanese-Llama-2-13b-fast-instruct",
+		"dtype": "int4",
+		"is_messages": false,
+		"template": "{bos_token}[INST] <<SYS>>\n{instruction}\n<</SYS>>\n{input} [/INST]",
+		"max_new_tokens": 256,
+		"temperature": 1.0,
+		"top_p": 0.9,
+		"top_k": 40,
+		"repetition_penalty": 1.1
+	},
+    "tokyotech-llm/Swallow-13b-instruct-hf": {
+		"model": "tokyotech-llm/Swallow-13b-instruct-hf",
+		"dtype": "int4",
+		"is_messages": false,
+		"template":  "以下に、あるタスクを説明する指示があり、それに付随する入力が更なる文脈を提供しています。リクエストを適切に完了するための回答を記述してください。\n\n### 指示:\n{instruction}\n\n### 入力:\n{input}\n\n### 応答:",
+		"max_new_tokens": 256,
+		"temperature": 0.99,
+		"top_p": 0.95,
+		"top_k": 40,
+		"repetition_penalty": 1.1
+	},
+    "rinna/nekomata-14b-instruction": {
+		"model": "rinna/nekomata-14b-instruction",
+		"dtype": "int4",
+		"is_messages": false,
+		"template":  "以下は、タスクを説明する指示と、文脈のある入力の組み合わせです。要求を適切に満たす応答を書きなさい。\n\n### 指示:\n{instruction}\n\n### 入力:\n{input}\n\n### 応答:",
+		"max_new_tokens": 256,
+		"temperature": 0.5,
+		"top_p": 0.95,
+		"top_k": 40,
+		"repetition_penalty": 1.1
+	},
+    "cyberagent/calm2-7b-chat": {
+		"model": "cyberagent/calm2-7b-chat",
+		"dtype": "int4",
+		"is_messages": false,
+		"template":  "{instruction}\nUSER: {input}\nASSISTANT: ",
+		"max_new_tokens": 256,
+		"temperature": 0.8,
+		"top_p": 0.9,
+		"top_k": 40,
+		"repetition_penalty": 1.1
+	},
+    "llm-jp/llm-jp-13b-instruct-full-jaster-dolly-oasst-v1.0": {
+		"model": "llm-jp/llm-jp-13b-instruct-full-jaster-dolly-oasst-v1.0",
+		"dtype": "int4",
+		"is_messages": false,
+		"template":  "{instruction}\n{input}\n### 回答：",
+		"max_new_tokens": 256,
+		"temperature": 0.7,
+		"top_p": 0.95,
+		"top_k": 40,
+		"repetition_penalty": 1.1
+	},
+    "stockmark/stockmark-13b-instruct": {
+		"model": "stockmark/stockmark-13b-instruct",
+		"dtype": "int4",
+		"is_messages": false,
+		"template":  "{instruction}\n\n### Input:\n{input}\n\n### Output:",
+		"max_new_tokens": 256,
+		"temperature": 0.7,
+		"top_p": 0.9,
+		"top_k": 40,
+		"repetition_penalty": 1.1
+	}
+}

models.py

@@ -0,0 +1,19 @@
+import json
+
+def load():
+    with open('models.json', 'r', encoding='utf-8') as f:
+        models = json.load(f)
+
+    return models
+
+def get_examples():
+    examples = []
+    for model in models.keys():
+        examples.append(list(models[model].values()))
+
+    return examples
+
+def get_head_options():
+    return models[list(models.keys())[0]]
+
+models = load()

requirements.txt

@@ -0,0 +1,12 @@
+fastapi
+uvicorn
+transformers
+accelerate
+sentencepiece
+bitsandbytes
+scipy
+tiktoken
+einops
+transformers_stream_generator
+protobuf
+python-multipart

venv.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/bash
+
+python3 -m venv venv
+curl -kL https://bootstrap.pypa.io/get-pip.py | venv/bin/python
+
+venv/bin/python -m pip install gradio
+venv/bin/python -m pip install -r requirements.txt