update model init with float16

README.md

@@ -32,11 +32,15 @@ Use the code below to get started with the model.
 ```python
 import requests
 
+import torch
 from PIL import Image
 from transformers import AutoProcessor, AutoModelForCausalLM 
 
 
-model = AutoModelForCausalLM.from_pretrained("microsoft/Florence-2-large", trust_remote_code=True)
+device = "cuda:0" if torch.cuda.is_available() else "cpu"
+torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
+
+model = AutoModelForCausalLM.from_pretrained("microsoft/Florence-2-large", torch_dtype=torch_dtype, trust_remote_code=True).to(device)
 processor = AutoProcessor.from_pretrained("microsoft/Florence-2-large", trust_remote_code=True)
 
 prompt = "<OD>"
@@ -44,7 +48,7 @@ prompt = "<OD>"
 url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/car.jpg?download=true"
 image = Image.open(requests.get(url, stream=True).raw)
 
-inputs = processor(text=prompt, images=image, return_tensors="pt")
+inputs = processor(text=prompt, images=image, return_tensors="pt").to(device, torch_dtype)
 
 generated_ids = model.generate(
     input_ids=inputs["input_ids"],
@@ -74,11 +78,14 @@ First, let's define a function to run a prompt.
 ```python
 import requests
 
+import torch
 from PIL import Image
 from transformers import AutoProcessor, AutoModelForCausalLM 
 
+device = "cuda:0" if torch.cuda.is_available() else "cpu"
+torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
 
-model = AutoModelForCausalLM.from_pretrained("microsoft/Florence-2-large", trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained("microsoft/Florence-2-large", torch_dtype=torch_dtype, trust_remote_code=True).to(device)
 processor = AutoProcessor.from_pretrained("microsoft/Florence-2-large", trust_remote_code=True)
 
 url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/car.jpg?download=true"
@@ -89,7 +96,7 @@ def run_example(task_prompt, text_input=None):
         prompt = task_prompt
     else:
         prompt = task_prompt + text_input
-    inputs = processor(text=prompt, images=image, return_tensors="pt")
+    inputs = processor(text=prompt, images=image, return_tensors="pt").to(device, torch_dtype)
     generated_ids = model.generate(
       input_ids=inputs["input_ids"],
       pixel_values=inputs["pixel_values"],

config.json

@@ -79,7 +79,7 @@
     "image_feature_source": ["spatial_avg_pool", "temporal_avg_pool"]
   },
   "vocab_size": 51289,
-  "torch_dtype": "float32",
+  "torch_dtype": "float16",
   "transformers_version": "4.41.0.dev0",
   "is_encoder_decoder": true
 }

processing_florence2.py

@@ -324,7 +324,7 @@ class Florence2Processor(ProcessorMixin):
         if task_answer_post_processing_type == 'pure_text':
             final_answer = task_answer
             # remove the special tokens
-            final_answer = final_answer.replace('<s>', '').replace('</s>', '\n')
+            final_answer = final_answer.replace('<s>', '').replace('</s>', '')
         elif task_answer_post_processing_type in ['od', 'description_with_bboxes', 'bboxes']:
             od_instances = task_answer
             bboxes_od = [_od_instance['bbox'] for _od_instance in od_instances]