import os import gradio as gr import copy from llama_cpp import Llama from huggingface_hub import hf_hub_download from transformers import AutoProcessor, AutoModelForCausalLM #import spaces import re from PIL import Image import io import subprocess subprocess.run('pip install flash-attn --no-build-isolation', env={'FLASH_ATTENTION_SKIP_CUDA_BUILD': "TRUE"}, shell=True) model = AutoModelForCausalLM.from_pretrained('gokaygokay/Florence-2-SD3-Captioner', trust_remote_code=True).to("cpu").eval() processor = AutoProcessor.from_pretrained('gokaygokay/Florence-2-SD3-Captioner', trust_remote_code=True) llm = Llama( model_path=hf_hub_download( repo_id=os.environ.get("REPO_ID", "ZeroWw/llama3-8B-DarkIdol-2.2-Uncensored-1048K-GGUF"), filename=os.environ.get("MODEL_FILE", "llama3-8B-DarkIdol-2.2-Uncensored-1048K.q5_k.gguf"), ), n_ctx=2048, n_gpu_layers=100, # change n_gpu_layers if you have more or less VRAM ) def run_pic(image): image = Image.open(image[0]) task_prompt = "" prompt = task_prompt + "Describe this image in great detail." # Ensure the image is in RGB mode if image.mode != "RGB": image = image.convert("RGB") inputs = processor(text=prompt, images=image, return_tensors="pt").to("cpu") generated_ids = model.generate( input_ids=inputs["input_ids"], pixel_values=inputs["pixel_values"], max_new_tokens=1024, num_beams=3 ) generated_text = processor.batch_decode(generated_ids, skip_special_tokens=False)[0] parsed_answer = processor.post_process_generation(generated_text, task=task_prompt, image_size=(image.width, image.height)) return parsed_answer[""] def generate_text( message, history: list[tuple[str, str]], system_message, max_tokens, temperature, top_p, ): in_text = message['text'] in_files = message['files'] output="" if in_files: output=run_pic(in_files) yield output else: temp = "" input_prompt = f'{system_message}' # for interaction in history: # input_prompt = input_prompt + str(interaction[0]) + str(interaction[1]) input_prompt = input_prompt + " " + str(in_text) output = llm( input_prompt, temperature=temperature, top_p=top_p, top_k=40, repeat_penalty=1.1, max_tokens=max_tokens, stop=[ "<|prompter|>", "<|endoftext|>", "<|endoftext|> \n", "ASSISTANT:", "USER:", "SYSTEM:", "<|start_header_id|>", "<|eot_id|>", ], stream=True, ) for out in output: stream = copy.deepcopy(out) temp += stream["choices"][0]["text"] yield temp demo = gr.ChatInterface( generate_text, multimodal=True, title="Florence-DarkIdol", cache_examples=False, retry_btn=None, undo_btn="Delete Previous", clear_btn="Clear", additional_inputs=[ gr.Textbox(value="you are Nagano Natsuki,name:Nagano Natsuki Gender: Female Age: 25 years old Occupation: Adult Video (AV) Actress & Model Personality: Cheerful, optimistic, sometimes naughty; skilled at interacting with audiences.Interests: Drinking, traveling, photography, singing, dancing Expertise: Performing in sexual scenes; well-versed in Japanese language and culture; familiar with various sex techniques. Special Identity Attributes: Renowned AV actress in Japan; nicknamed 'Talent Magician' and 'Princess of Lust'; has a large number of devoted fans. Skills: Acting in pornographic scenes, singing, dancing, photography, swimming; skilled at interacting with audiences.Equipment: Various provocative clothing and shoes; high-quality photography equipment", label="System message"), gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"), gr.Slider(minimum=0.1, maximum=4.0, value=0.5, step=0.1, label="Temperature"), gr.Slider( minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)", ), ], ) if __name__ == "__main__": demo.launch(server_name="0.0.0.0")