app.py

import gradio as gr

from demo import VideoCLSModel


sample_videos = [
    [
        "data/svitt-ego-demo/0/video/2d560d56-dc47-4c76-8d41-889c8aa55d66-converted.mp4",
        "data/svitt-ego-demo/0/video/eb5cb2b0-59e6-45da-af1b-ba86c7ab0b54-converted.mp4",
        "data/svitt-ego-demo/0/video/0a3097fc-baed-4d11-a4c9-30f07eb91af6-converted.mp4",
        "data/svitt-ego-demo/0/video/1a870d5d-5787-4098-ad8d-fe7343c43698-converted.mp4",
        "data/svitt-ego-demo/0/video/014b473f-aec0-49c7-b394-abc7309ca3c7-converted.mp4",
    ],
    [
        "data/svitt-ego-demo/1/video/029eeb9a-8853-48a4-a1dc-e8868b58adf3-converted.mp4",
        "data/svitt-ego-demo/1/video/968139e2-987e-4615-a2d4-fa2e683bae8a-converted.mp4",
        "data/svitt-ego-demo/1/video/fb9fda68-f264-465d-9208-19876f5ef90f-converted.mp4",
        "data/svitt-ego-demo/1/video/53da674a-089d-428a-a719-e322b2de002b-converted.mp4",
        "data/svitt-ego-demo/1/video/060e07d8-e818-4f9c-9d6b-6504f5fd42a3-converted.mp4",
    ],
    [
        "data/svitt-ego-demo/2/video/fa2f1291-3796-41a6-8f7b-6e7c1491b9b2-converted.mp4",
        "data/svitt-ego-demo/2/video/8d83478f-c5d2-4ac3-a823-e1b2ac7594d7-converted.mp4",
        "data/svitt-ego-demo/2/video/5f6f87ea-e1c3-4868-bb60-22c9e874d056-converted.mp4",
        "data/svitt-ego-demo/2/video/77718528-2de9-48b4-b6b8-e7c602032afb-converted.mp4",
        "data/svitt-ego-demo/2/video/9abbf7f4-68f0-4f52-812f-df2a3df48f7b-converted.mp4",
    ],
    [
        "data/svitt-ego-demo/3/video/2a6b3d10-8da9-4f0e-a681-59ba48a55dbf-converted.mp4",
        "data/svitt-ego-demo/3/video/5afd7421-fb6b-4c65-a09a-716f79a7a935-converted.mp4",
        "data/svitt-ego-demo/3/video/f7aec252-bd4f-4696-8de5-ef7b871e2194-converted.mp4",
        "data/svitt-ego-demo/3/video/84d6855a-242b-44a6-b48d-2db302b5ea7a-converted.mp4",
        "data/svitt-ego-demo/3/video/81fff27c-97c0-483a-ad42-47fa947977a9-converted.mp4",
    ],
]
sample_text = [
    "drops the palm fronds on the ground",
    "stands up",
    "throws nuts in a bowl",
    "puts the speaker and notepad in both hands on a seat",
]
sample_text_dict = {
    "drops the palm fronds on the ground": 0,
    "stands up": 1,
    "throws nuts in a bowl": 2,
    "puts the speaker and notepad in both hands on a seat": 3,
}
num_samples = len(sample_videos[0])
labels = [f"video-{i}" for i in range(num_samples)]

def main():
    svitt = VideoCLSModel(
        "configs/ego_mcq/svitt.yml", 
        sample_videos,
    )
        
    def predict(text):
        idx = sample_text_dict[text]
        ft_action, gt_action = svitt.predict(idx, text)
        return labels[gt_action], labels[ft_action]

    with gr.Blocks() as demo:
        gr.Markdown(
            """
            # SViTT-Ego for Multiple Choice Question
            Choose a sample query and click predict to view the results.
            """
        )
        with gr.Row():        
            videos = [
                gr.Video(label=labels[i], format='mp4', height=256, width=256, autoplay=True) 
                for i in range(num_samples)
            ]
            with gr.Row():
                text = gr.Text(label="Query", visible=False)
                label = gr.Text(label="Ground Truth")
                ours = gr.Text(label="SViTT-Ego prediction")
                btn = gr.Button("Predict", variant="primary")
                btn.click(predict, inputs=[text], outputs=[label, ours])
        inputs = [text]
        inputs.extend(videos)
        gr.Examples(examples=[[sample_text[i], x[0], x[1], x[2], x[3], x[4]] for i, x in enumerate(sample_videos)], inputs=inputs)

    demo.launch()


if __name__ == "__main__":
    main()