utils.py

import seaborn as sns
from PIL import Image, ImageDraw, ImageFont
import matplotlib.font_manager
import spacy
import re
import base64
import time
import re
from PIL import Image
import base64
import hashlib
import os

nlp = spacy.load("en_core_web_sm-3.6.0")

def process_image_without_resize(image_prompt):
    image = Image.open(image_prompt)
    print(f"height:{image.height}, width:{image.width}")
    timestamp = time.time()
    file_ext = os.path.splitext(image_prompt)[1]
    filename = f"examples/{timestamp}{file_ext}"
    filename_grounding = f"examples/{timestamp}_grounding{file_ext}"
    image.save(filename)
    print(f"temporal filename {filename}")
    with open(filename, "rb") as image_file:
        bytes = base64.b64encode(image_file.read())
        encoded_img = str(bytes, encoding='utf-8')
        image_hash = hashlib.sha256(bytes).hexdigest()
        os.remove(filename)
        return image, encoded_img, image_hash, filename_grounding


def is_chinese(text):
    zh_pattern = re.compile(u'[\u4e00-\u9fa5]+')
    return zh_pattern.search(text)


def draw_boxes(image, boxes, texts, output_fn='output.png'):
    box_width = 5
    color_palette = sns.color_palette("husl", len(boxes))
    colors = [(int(r*255), int(g*255), int(b*255)) for r, g, b in color_palette]

    width, height = image.size
    absolute_boxes = [[(int(box[0] * width), int(box[1] * height), int(box[2] * width), int(box[3] * height)) for box in b] for b in boxes]
    
    overlay = Image.new('RGBA', image.size, (255, 255, 255, 0))
    draw = ImageDraw.Draw(overlay)
    font_path = sorted(matplotlib.font_manager.findSystemFonts(fontpaths=None, fontext='ttf'))[0]
    font = ImageFont.truetype(font_path, size=26)

    for box, text, color in zip(absolute_boxes, texts, colors):
        for b in box:
            draw.rectangle(b, outline=color, width=box_width)
            if not text:
                continue
            splited_text = text.split('\n')
            num_lines = len(splited_text)
            text_width, text_height = font.getbbox(splited_text[0])[-2:]
            y_start = b[3] - text_height * num_lines - box_width
            if b[2] - b[0] < 100 or b[3] - b[1] < 100:
                y_start = b[3]
            for i, line in enumerate(splited_text):
                text_width, text_height = font.getbbox(line)[-2:]
                x = b[0] + box_width
                y = y_start + text_height * i
                draw.rectangle([x, y, x+text_width, y+text_height], fill=(128, 128, 128, 160))
                draw.text((x, y), line, font=font, fill=(255, 255, 255))
    img_with_overlay = Image.alpha_composite(image.convert('RGBA'), overlay).convert('RGB')
    img_with_overlay.save(output_fn)

def boxstr_to_boxes(box_str):
    boxes = [[int(y)/1000 for y in x.split(',')] for x in box_str.split(';') if x.replace(',', '').isdigit()]
    return boxes

def text_to_dict(text):
    doc = nlp(text)

    box_matches = list(re.finditer(r'\[\[([^\]]+)\]\]', text))
    box_positions = [match.start() for match in box_matches]

    noun_phrases = []
    boxes = []

    for match, box_position in zip(box_matches, box_positions):
        nearest_np_start = max([0] + [chunk.start_char for chunk in doc.noun_chunks if chunk.end_char <= box_position])
        noun_phrase = text[nearest_np_start:box_position].strip()
        if noun_phrase and noun_phrase[-1] == '?':
            noun_phrase = text[:box_position].strip()
        box_string = match.group(1)
        
        noun_phrases.append(noun_phrase)
        boxes.append(boxstr_to_boxes(box_string))

    pairs = []
    for noun_phrase, box_string in zip(noun_phrases, boxes):
        pairs.append((noun_phrase.lower(), box_string))
    return dict(pairs)

def parse_response(img, response, output_fn='output.png'):
    img = img.convert('RGB')
    width, height = img.size
    ratio = min(1920 / width, 1080 / height)
    new_width = int(width * ratio)
    new_height = int(height * ratio)
    new_img = img.resize((new_width, new_height), Image.LANCZOS)
    pattern = r"\[\[(.*?)\]\]"
    positions = re.findall(pattern, response)
    boxes = [[[int(y) for y in x.split(',')] for x in pos.split(';') if x.replace(',', '').isdigit()] for pos in positions]
    dic = text_to_dict(response)
    if not dic:
        texts = []
        boxes = []
    else:
        texts, boxes = zip(*dic.items())
    draw_boxes(new_img, boxes, texts, output_fn=output_fn)

def postprocess_text(template, text):
    quoted_text = f'"{text.strip()}"'
    return template.replace("<TASK>", quoted_text).strip() if template != "" else text.strip()

    
# The templates is for CogAgent_Agent Template
templates_agent_cogagent = [
    "Can you advise me on how to <TASK>?",
    "I'm looking for guidance on how to <TASK>.",
    "What steps do I need to take to <TASK>?",
    "Could you provide instructions for <TASK>?",
    "I'm wondering what the process is for <TASK>.",
    "How can I go about <TASK>?",
    "I need assistance with planning to <TASK>.",
    "Do you have any recommendations for <TASK>?",
    "Please share some tips for <TASK>.",
    "I'd like to know the best way to <TASK>.",
    "What's the most effective way to <TASK>?",
    "I'm seeking advice on accomplishing <TASK>.",
    "Could you guide me through the steps to <TASK>?",
    "I'm unsure how to start with <TASK>.",
    "Is there a strategy for successfully <TASK>?",
    "What's the proper procedure for <TASK>?",
    "How should I prepare for <TASK>?",
    "I'm not sure where to begin with <TASK>.",
    "I need some insights on <TASK>.",
    "Can you explain how to tackle <TASK>?",
    "I'm interested in the process of <TASK>.",
    "Could you enlighten me on <TASK>?",
    "What are the recommended steps for <TASK>?",
    "Is there a preferred method for <TASK>?",
    "I'd appreciate your advice on <TASK>.",
    "Can you shed light on <TASK>?",
    "What would be the best approach to <TASK>?",
    "How do I get started with <TASK>?",
    "I'm inquiring about the procedure for <TASK>.",
    "Could you share your expertise on <TASK>?",
    "I'd like some guidance on <TASK>.",
    "What's your recommendation for <TASK>?",
    "I'm seeking your input on how to <TASK>.",
    "Can you provide some insights into <TASK>?",
    "How can I successfully accomplish <TASK>?",
    "What steps are involved in <TASK>?",
    "I'm curious about the best way to <TASK>.",
    "Could you show me the ropes for <TASK>?",
    "I need to know how to go about <TASK>.",
    "What are the essential steps for <TASK>?",
    "Is there a specific method for <TASK>?",
    "I'd like to get some advice on <TASK>.",
    "Can you explain the process of <TASK>?",
    "I'm looking for guidance on how to approach <TASK>.",
    "What's the proper way to handle <TASK>?",
    "How should I proceed with <TASK>?",
    "I'm interested in your expertise on <TASK>.",
    "Could you walk me through the steps for <TASK>?",
    "I'm not sure where to begin when it comes to <TASK>.",
    "What should I prioritize when doing <TASK>?",
    "How can I ensure success with <TASK>?",
    "I'd appreciate some tips on <TASK>.",
    "Can you provide a roadmap for <TASK>?",
    "What's the recommended course of action for <TASK>?",
    "I'm seeking your guidance on <TASK>.",
    "Could you offer some suggestions for <TASK>?",
    "I'd like to know the steps to take for <TASK>.",
    "What's the most effective way to achieve <TASK>?",
    "How can I make the most of <TASK>?",
    "I'm wondering about the best approach to <TASK>.",
    "Can you share your insights on <TASK>?",
    "What steps should I follow to complete <TASK>?",
    "I'm looking for advice on <TASK>.",
    "What's the strategy for successfully completing <TASK>?",
    "How should I prepare myself for <TASK>?",
    "I'm not sure where to start with <TASK>.",
    "What's the procedure for <TASK>?",
    "Could you provide some guidance on <TASK>?",
    "I'd like to get some tips on how to <TASK>.",
    "Can you explain how to tackle <TASK> step by step?",
    "I'm interested in understanding the process of <TASK>.",
    "What are the key steps to <TASK>?",
    "Is there a specific method that works for <TASK>?",
    "I'd appreciate your advice on successfully completing <TASK>.",
    "Can you shed light on the best way to <TASK>?",
    "What would you recommend as the first step to <TASK>?",
    "How do I initiate <TASK>?",
    "I'm inquiring about the recommended steps for <TASK>.",
    "Could you share some insights into <TASK>?",
    "I'm seeking your expertise on <TASK>.",
    "What's your recommended approach for <TASK>?",
    "I'd like some guidance on where to start with <TASK>.",
    "Can you provide recommendations for <TASK>?",
    "What's your advice for someone looking to <TASK>?",
    "I'm seeking your input on the process of <TASK>.",
    "How can I achieve success with <TASK>?",
    "What's the best way to navigate <TASK>?",
    "I'm curious about the steps required for <TASK>.",
    "Could you show me the proper way to <TASK>?",
    "I need to know the necessary steps for <TASK>.",
    "What's the most efficient method for <TASK>?",
    "I'd appreciate your guidance on <TASK>.",
    "Can you explain the steps involved in <TASK>?",
    "I'm looking for recommendations on how to approach <TASK>.",
    "What's the right way to handle <TASK>?",
    "How should I manage <TASK>?",
    "I'm interested in your insights on <TASK>.",
    "Could you provide a step-by-step guide for <TASK>?",
    "I'm not sure how to start when it comes to <TASK>.",
    "What are the key factors to consider for <TASK>?",
    "How can I ensure a successful outcome with <TASK>?",
    "I'd like some tips and tricks for <TASK>.",
    "Can you offer a roadmap for accomplishing <TASK>?",
    "What's the preferred course of action for <TASK>?",
    "I'm seeking your expert advice on <TASK>.",
    "Could you suggest some best practices for <TASK>?",
    "I'd like to understand the necessary steps to complete <TASK>.",
    "What's the most effective strategy for <TASK>?",
]

template_grounding_cogvlm = [
    "Where is <TASK>?",
    "Where is <TASK> in the image?",
    "Where is <TASK>? answer in [[x0,y0,x1,y1]] format.",
    "Can you point out <TASK> in the image and provide the bounding boxes of its location?",
    "Help me to locate <TASK> in and give me its bounding boxes, please.",
    "In the given, could you find and tell me the bounding boxes of <TASK>?",
    "Guide me to the location of <TASK> within the image by providing its bounding boxes.",
    "I'd like to know the exact bounding boxes of <TASK> in the photo.",
    "Would you kindly provide the bounding boxes of <TASK> located in the picture?",
    "Can you find <TASK> in and give me the bounding boxes of where it is located?",
    "I'm trying to locate <TASK> in. Can you determine its bounding boxes for me?",
    "What are the bounding boxes of <TASK> in the image?",
    "Can you disclose the position of <TASK> in the photograph by stating its bounding boxes?",
    "In, could you let me know the location of <TASK> in the form of bounding boxes?",
    "I need the bounding boxes of <TASK> in, can you please assist me with that?",
    "Where in is <TASK> located? Provide me with its bounding boxes, please.",
    "May I have the bounding boxes of <TASK>?",
    "In the photograph, could you pinpoint the location of <TASK> and tell me its bounding boxes?",
    "Can you please search and find <TASK> in, then let me know its bounding boxes?",
    "Please, point out the position of <TASK> in the image by giving its bounding boxes.",
    "What are the exact bounding boxes of <TASK> in the provided picture?",
    "Detect the location of <TASK> in and share the bounding boxes with me, please.",
    "In the picture, I'd like you to locate <TASK> and provide its coordinates.",
    "Please indicate the location of <TASK> in the photo by giving bounding boxes.",
    "Find <TASK> in and share its coordinates with me.",
    "Could you please help me find the bounding boxes of <TASK> in the image?",
    "I am looking for the position of <TASK> in. Can you provide its bounding boxes?",
    "In the image, can you locate <TASK> and let me know its coordinates?",
    "I'd appreciate if you could find and tell me the bounding boxes of <TASK>.",
    "In, I need the bounding box bounding boxes of <TASK>.",
    "Point me to the location of <TASK> in the picture by providing its bounding boxes.",
    "Could you trace <TASK> in and tell me its bounding boxes?",
    "Can you assist me in locating <TASK> in, and then provide its bounding boxes?",
    "I'm curious, what are the bounding boxes of <TASK> in the photo?",
    "Kindly share the bounding boxes of <TASK> located in the image.",
    "I would like to find <TASK> in. Can you give me its bounding boxes?",
    "Can you spot <TASK> in and disclose its bounding boxes to me?",
    "Please, reveal the location of <TASK> in the provided photograph as coordinates.",
    "Help me locate and determine the bounding boxes of <TASK>.",
    "I request the bounding boxes of <TASK> in the image.",
    "In the given, can you find <TASK> and tell me its bounding boxes?",
    "I need to know the position of <TASK> in as bounding boxes.",
    "Locate <TASK> in and provide its bounding boxes, please.",
    "Assist me in finding <TASK> in the photo and provide the bounding box bounding boxes.",
    "In, can you guide me to the location of <TASK> by providing bounding boxes?",
    "I'd like the bounding boxes of <TASK> as it appears in the image.",
    "What location does <TASK> hold in the picture? Inform me of its bounding boxes.",
    "Identify the position of <TASK> in and share its bounding boxes.",
    "I'd like to request the bounding boxes of <TASK> within the photo.",
    "How can I locate <TASK> in the image? Please provide the bounding boxes.",
    "I am interested in knowing the bounding boxes of <TASK> in the picture.",
    "Assist me in locating the position of <TASK> in the photograph and its bounding box bounding boxes.",
    "In the image, I need to find <TASK> and know its bounding boxes. Can you please help?"
    "Can you give me a description of the region <TASK> in image?",
    "In the provided image, would you mind describing the selected area <TASK>?",
    "I need details about the area <TASK> located within image.",
    "Could you please share some information on the region <TASK> in this photograph?",
    "Describe what's happening within the coordinates <TASK> of the given image.",
    "What can you tell me about the selected region <TASK> in the photo?",
    "Please, can you help me understand what's inside the region <TASK> in image?",
    "Give me a comprehensive description of the specified area <TASK> in the picture.",
    "I'm curious about the area <TASK> in the following image. Can you describe it?",
    "Please elaborate on the area with the coordinates <TASK> in the visual.",
    "In the displayed image, help me understand the region defined by <TASK>.",
    "Regarding the image, what's going on in the section <TASK>?",
    "In the given photograph, can you explain the area with coordinates <TASK>?",
    "Kindly describe what I should be seeing in the area <TASK> of image.",
    "Within the input image, what can be found in the region defined by <TASK>?",
    "Tell me what you see within the designated area <TASK> in the picture.",
    "Please detail the contents of the chosen region <TASK> in the visual input.",
    "What's inside the area <TASK> of the provided graphic?",
    "I'd like some information about the specific region <TASK> in the image.",
    "Help me understand the details within the area <TASK> in photograph.",
    "Can you break down the region <TASK> in the image for me?",
    "What is taking place within the specified area <TASK> in this capture?",
    "Care to elaborate on the targeted area <TASK> in the visual illustration?",
    "What insights can you provide about the area <TASK> in the selected picture?",
    "What does the area <TASK> within the given visual contain?",
    "Analyze and describe the region <TASK> in the included photo.",
    "Please provide details for the area marked as <TASK> in this photographic.",
    "For the image, can you assess and describe what's happening at <TASK>?",
    "Fill me in about the selected portion <TASK> within the presented image.",
    "In the image, elaborate on the details found within the section <TASK>.",
    "Please interpret and describe the area <TASK> inside the given picture.",
    "What information can you give me about the coordinates <TASK> in image?",
    "Regarding the coordinates <TASK> in image, can you provide a description?",
    "In the photo, can you delve into the details of the region <TASK>?",
    "Please provide insights on the specified area <TASK> within the graphic.",
    "Detail the chosen region <TASK> in the depicted scene.",
    "Can you discuss the entities within the region <TASK> of image?",
    "I'd appreciate a breakdown of the area <TASK> in the displayed image.",
    "What's the story in the section <TASK> of the included visual?",
    "Please enlighten me about the region <TASK> in the given photo.",
    "Offer a thorough description of the area <TASK> within the illustration.",
    "What can you share about the area <TASK> in the presented image?",
    "Help me grasp the context of the region <TASK> within image.",
    "Kindly give an overview of the section <TASK> in photo.",
    "What details can you provide about the region <TASK> in the snapshot?",
    "Can you divulge the contents of the area <TASK> within the given image?",
    "In the submitted image, please give a synopsis of the area <TASK>.",
    "In the image, please describe the bounding box <TASK>.",
    "Please describe the region <TASK> in the picture.",
    "Describe the bbox <TASK> in the provided photo.",
    "What can you tell me about the area <TASK> within the image?",
    "Could you give me a description of the rectangular region <TASK> found in?",
    "In, what elements can be found within the coordinates <TASK>?",
    "Please provide details for the area within the bounding box <TASK> in.",
    "Can you generate a description for the selected region <TASK> in the image?",
    "Kindly describe the objects or scenery in the bounding box <TASK> within.",
    "What details can you provide for the rectangle defined by the coordinates <TASK> in?",
    "In relation to the picture, please describe the content of the area marked by <TASK>.",
    "I'd like to know more about the area <TASK> in the given image. Can you describe it?",
    "Can you help me by describing the part of that lies within the bounding box <TASK>?",
    "What's happening in the section of the photo enclosed by the coordinates <TASK>?",
    "Describe the image content present in the specified rectangular area <TASK> of.",
    "Please provide information about the area within the bounding box <TASK> in the picture.",
    "Could you offer a description of the contents in the selected area <TASK> of the image?",
    "I'm curious about the area <TASK> in. Can you provide a description of it?",
    "What can be observed in the rectangular region <TASK> in the photograph?",
    "Please explain what is contained in the portion of defined by the box <TASK>.",
    "In the photograph, can you describe the objects or scenery enclosed by <TASK>?",
    "Can you give a brief explanation of the specified area <TASK> in the image?",
    "What does the area <TASK> look like in the context of the image?",
    "Could you please describe the contents of the bounding box <TASK> in the given image?",
    "I would like to know more about the rectangular region <TASK> within the picture. Can you describe it?",
    "Please tell me about the area <TASK> in the image. What does it contain?",
    "Help me understand what's happening in the selected bounding box <TASK> within.",
    "Can you provide a description of the area <TASK> in the image?",
    "What sort of things can be seen in the region <TASK> of the photo?",
    "Describe what can be found within the bounds of <TASK> in the image.",
    "In, can you paint a picture of the area enclosed by coordinates <TASK>?",
    "Please provide a detailed account of the area covered by the bounding box <TASK> in.",
    "Give me a vivid description of what's happening in the area <TASK> within the snapshot.",
    "In the image, what do you observe within the rectangular box defined by the coordinates <TASK>?",
    "Could you give me a breakdown of the content in the specified area <TASK> of the picture?",
    "Please elucidate the area<TASK> of the image.",
    "I'd appreciate it if you could describe the portion of that lies within the rectangle <TASK>.",
    "Can you share some insights about the rectangular region <TASK> in the image?",
    "Help me visualize the section of the photo enclosed by the bounding box <TASK>.",
    "Would you kindly provide a description for the content within the rectangular area <TASK> of?",
    "In, can you tell me more about the area specified by the bounding box <TASK>?",
    "Please describe what can be seen in the rectangular region <TASK> of the image.",
    "Can you analyze the content of the area <TASK> within the photograph?",
    "In the provided image, please explain the content within the region <TASK>.",
    "I'm interested in the selected rectangle <TASK> in. Can you tell me more about it?",
    "Explain what can be found in the bounding box <TASK> in the context of the image.",
    "Kindly share your observations about the rectangular region <TASK> within.",
    "I'd like a thorough description of the area <TASK> in the image.",
    "Could you please provide a description of the rectangular area <TASK> in?",
    "Please describe the section of the picture defined by the bbox <TASK>.",
    "Tell me more about the scenery or objects within the rectangular region <TASK> in.",
    "Would you kindly describe the content of the area enclosed by <TASK> in the image?",
    "Help me understand the objects or scenery within the bounding box <TASK> in the image.",
    "I would like to know about the section of the image enclosed by the rectangle <TASK>. Can you describe it?",
    "Describe the selected rectangular area <TASK> in the photo.",
    "Tell me about the region <TASK> of the image.",
    "I request a description of the area <TASK> in the picture.",
    "Can you elaborate on the content of the bounding box <TASK> in?",
    "Please share details about the rectangular region <TASK> within the image.",
    "What can I find in the bbox <TASK> of the provided image?",
    "In the image, could you provide a description for the coordinates <TASK>?",
    "Could you tell me more about the area <TASK> in the snapshot?",
    "Fill me in on the details of the rectangular box <TASK> within the image.",
    "What's going on in the section of contained within the bounding box <TASK>?",
    "I would like a description of the content within the bbox <TASK> in.",
    "Please enlighten me about the area <TASK> in the photograph.",
    "Can you give me a visual rundown of the area <TASK> in?",
    "Describe the visual elements within the selected area <TASK> of the image.",
    "Tell me what you see in the area <TASK> within the context of the image.",
    "Explain the content within the rectangular region <TASK> of the image.",
    "I'd like some information about the bounding box <TASK> in the photo.",
    "What is happening within the rectangle defined by coordinates <TASK> in the image?",
    "Please describe the content within the area <TASK> displayed in the image.",
    "What can be seen in the bounding box <TASK> in the context of the provided image?",
    "Share some details about the objects or environment within the bounding box <TASK> in.",
    "Please describe the area <TASK> in the image for me.",
    "Can you generate a description of the contents within the selected region <TASK> in?",
    "What objects or scenery can be found in the area <TASK> in the image?",
    "Please tell me more about the rectangular section <TASK> in the photo.",
    "Could you describe the content of the bbox <TASK> in the image?",
    "What does the selected region <TASK> in the image encompass?",
    "I am interested in the region <TASK> of the image; please describe it.",
    "Can you provide some context for the area <TASK> within the picture?",
    "Please give me some details about the rectangle <TASK> in the image.",
    "In the photo, what can you see within the region defined by the bounding box <TASK>?",
    "I would like a detailed description of the portion of enclosed by the bbox <TASK>.",
    "Please help me understand the content present within the rectangle <TASK> in.",
    "Would you mind describing the rectangular area <TASK> in the provided image?"
]