Update app.py

app.py

@@ -1,39 +1,212 @@
-import gradio as gr
+import streamlit as st
+from PIL import Image
+import os
 from deepforest import main
+from deepforest import get_data
 import matplotlib.pyplot as plt
+# from predict import extract_features, predict_similarity, compare_features, extract_features_cp
+import os, re
+import streamlit as st
+import pandas as pd
+from PIL import Image
+import tempfile
+from inference import split_image_from_dataframe
+from datetime import datetime
+from predict_vit import extract_features, predict_similarity, compare_features, extract_features_cp
+from rag import generate_image, setup_client, setup_retriever
+from predict_copy import extract_features_with_augmentation, extract_features_with_augmentation_cp
+
+
+import rasterio
+import geopandas as gpd
 
-# Initialize the deepforest model and use the released version
 model = main.deepforest()
 model.use_release()
 
-def predict_and_visualize(image):
+# Set the page configuration
+st.set_page_config(page_title="Wise-Vision", page_icon=":deciduous_tree:")
+
+# Title and description
+st.title("🌳 Wise-Vision")
+st.subheader("AI + Environment Hackathon 2024")
+
+# Sidebar information
+st.sidebar.title("About")
+st.sidebar.info(
     """
-    Function to predict and visualize the image using deepforest model.
-    
-    Args:
-    - image: An image array.
-    
-    Returns:
-    - An image with predictions visualized.
+    This app is designed for the AI + Environment Hackathon 2024.
+    Upload a panoramic image and specify a folder path to detect tree species in the image.
+    Upload a word file to integrate knowledge into the image.
+    Output will be a panoramic image with identified trees and knowledge symbols.
+    """
+)
+
+st.sidebar.title("Contact")
+st.sidebar.info(
     """
-    # Predict image and return plot. Since Gradio passes image as array, save it temporarily.
-    temp_path = "/tmp/uploaded_image.png"
-    plt.imsave(temp_path, image)
-    img = model.predict_image(path=temp_path, return_plot=True)
+    For more information, contact us at:
+    [raj.[email protected]]
+    """
+)
+
+
+script_dir = os.path.dirname(os.path.abspath(__file__))
+
+# Create a new folder within the script directory for storing cropped images
+timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+output_folder_name = f"output_{timestamp}"
+output_image_folder = os.path.join(script_dir, output_folder_name)
+os.makedirs(output_image_folder, exist_ok=True)
+output_image_folder = os.path.abspath(output_image_folder)
+# Define paths for the image and Excel file within the new folder
+cropped_image_path = os.path.join(output_image_folder, f"panoramic_{timestamp}.png")
+excel_output_path = os.path.join(output_image_folder, f"results_{timestamp}.xlsx")
+
+# Input: Upload panoramic image
+uploaded_image = st.file_uploader("Upload a panoramic image", type=['png', 'jpeg', 'JPG'])
+
+# Input: Folder path for tree species detection
+
+def extract_treespecies_features(folder_path):
+    image_files = [os.path.join(folder_path, f) for f in os.listdir(folder_path) if f.endswith(('png', 'jpg', 'jpeg', '.JPG'))]
+
+    species_feature_list = [{"feature": extract_features_with_augmentation(file), "file_name": file} for file in image_files]
+    return species_feature_list
+
+
+# print(species_feature_list[:2])
+def perform_inference(cropped_images, species_feature_list, img_df):
+    st.success("Setting up OPENAI Client:")
+    client = setup_client()
+    st.success("Setting up knowledge database & BM25 retriever:")
+    retriever = setup_retriever()
+    st.success("Setting up BM25 Retriever:")
+    for img_idx, item in enumerate(cropped_images):
+        image = item["image"]
+        feature_cp = extract_features_with_augmentation_cp(image)
+        row_results = []
+        species_result = []
+        emoji = []
+        species_context = []
+        for idx, species in enumerate(species_feature_list):
+            # euclidean_dist, cos_sim = compare_features(feature_cp, species["feature"])
+        # print(f'Euclidean Distance: {euclidean_dist}')
+        # print(f'Cosine Similarity: {cos_sim}')
+
+        # Predict similarity
+            is_similar = predict_similarity(feature_cp, species["feature"], threshold=0.92)
+        # print(species)
+        # print(f'Are the images similar? {"Yes" if is_similar else "No"}')
+
+            result = "Yes" if is_similar else "No"
+
+            if result == "Yes":
+                item[f"result_{idx}"] = result
+                item[f"file_name_{idx}"] = species["file_name"]
+                row_results.append(species["file_name"])
+                # Regular expression to match the tree species name
+                species_pattern = r'identified_species\\([^\\]+) -'
+
+                # Search for the pattern in the file path
+                match = re.search(species_pattern, species["file_name"])
+
+                # Extract and print the tree species name if found
+                if match:
+                    tree_species = match.group(1)
+                    species_info = retriever.invoke(f"Scientific name:{tree_species}")
+                    
+                    ans = generate_image(species_info, client)
+                    emoji.append(ans)
+                    text_context = [doc.page_content for doc in species_info]
+                    text_context = ", ".join(text_context)
+                    species_context.append(text_context)
+                    # print(ans)
+                    species_result.append(tree_species)
+                    
+                else:
+                    print("Tree species name not found.")
+        img_df.at[img_idx, "species_identified"] = ", ".join(species_result) if species_result else "No similar species found"            
+        img_df.at[img_idx, "result_file_path"] = ", ".join(row_results) if row_results else ""
+        img_df.at[img_idx, "emoji"] = ", ".join(emoji) if emoji else ""
+        img_df.at[img_idx, "retreived context"] = ", ".join(species_context) if species_context else ""
+        
+        
+    return cropped_images
+
+
+# Function to simulate tree species detection
+
+# Display uploaded image and detected tree species
+if uploaded_image is not None:
+    with tempfile.NamedTemporaryFile(delete=False, suffix='.JPG') as temp_file:
+        temp_file.write(uploaded_image.read())
+        temp_file_path = temp_file.name
+    # Open and display the image
+    # image = Image.open(uploaded_image)
+    sample_image_path = get_data(temp_file_path)
+    boxes = model.predict_image(path=sample_image_path, return_plot=False)
+    img_actual = model.predict_image(path=sample_image_path, return_plot=True, color=(137, 0, 0), thickness=9)
+    st.image(img_actual, caption='Segmented Panoramic Image', channels ='RGB', use_column_width=True)
+    st.success("Sample Dataframe:")
+    st.dataframe(boxes.head())
+    plt.imshow(img_actual[:,:,::-1])
+    # plt.show(img[:,:,::-1])
+    plt.savefig(cropped_image_path)
+    # if st.button("Next Step"):
+  
+    accuracy_threshold = st.slider("Accuracy threshold for cropping images:",min_value=0.1, max_value=1.0, value=0.4)
+    images_list = split_image_from_dataframe(boxes, temp_file_path, output_folder_name)
+    image_width = 200
+    st.success("Sample Images:")
+    # Display the images in a row
+    col1, col2, col3 = st.columns(3)
+
+    with col1:
+        st.image(images_list[3]["image"], caption="Sample 1", width=image_width)
+
+    with col2:
+        st.image(images_list[4]["image"], caption="Sample 2", width=image_width)
+
+    with col3:
+        st.image(images_list[5]["image"], caption="Sample 3", width=image_width)
     
-    # Since the output is BGR and matplotlib (and hence Gradio) needs RGB, we convert the color scheme
-    img_rgb = img[:, :, ::-1]
+    folder_path = 'D:/Downloads/image/plant_images/plant_images/drone_igapo_flooded_forest/identified_species'
+
+    species_feature_list = extract_treespecies_features(folder_path)
+    final_result = perform_inference(images_list, species_feature_list, boxes)
+    st.success("Final Data:")
+    st.dataframe(boxes)
+    boxes.to_excel(excel_output_path)
+    for index, row in boxes.iterrows():
+        species_identified = row['species_identified']
+        if species_identified !="No similar species found":
+            cropped_image_path = row['cropped_image_path']
+            result_file_path = row['result_file_path']
+            if type(result_file_path) == list:
+                result_file_path = result_file_path[0]
+                
+
+            result_file_path = result_file_path.split(',')[0]
+            st.write(species_identified)
+            col1, col2 = st.columns(2)
+            with col1:
+                st.image(cropped_image_path, caption='Cropped Image')
+            with col2:
+                st.image(result_file_path, caption='Species Match')
+            
+            
+
+            
+            
+
     
-    # Return the RGB image
-    return img_rgb
-
-# Define the Gradio interface
-iface = gr.Interface(fn=predict_and_visualize,
-                     inputs=gr.Image(type="numpy", label="Upload Image"),
-                     outputs=gr.Image(label="Predicted Image"),
-                     title="DeepForest Tree Detection",
-                     examples=["./example.jpg"],
-                     description="Upload an image to detect trees using the DeepForest model.")
-
-# Launch the Gradio app
-iface.launch()
+    # Detect tree species
+    # detected_species = detect_tree_species(image, folder_path)
+    
+    # Display detected tree species
+    # st.write("### Detected Tree Species:")
+    # for species in detected_species:
+    #     st.write(f"- {species}")
+
+
+