Add NVLM

pages/28_NVEagle.py

@@ -159,7 +159,7 @@ with col2:
 with col3:
     if lang == "en":
         if st.button("Next paper", use_container_width=True):
-            switch_page("Home")
+            switch_page("NVLM")
     else:
         if st.button("Papier suivant", use_container_width=True):
-            switch_page("Home")
+            switch_page("NVLM")

pages/29_NVLM.py

@@ -0,0 +1,167 @@
+import streamlit as st
+from streamlit_extras.switch_page_button import switch_page
+
+
+translations = {
+'en': {'title': 'NVLM',
+    'original_tweet': 
+       """
+       [Original tweet](https://x.com/mervenoyann/status/1841098941900767323) (October 1st, 2024)
+       """,
+    'tweet_1':
+        """
+        NVIDIA just dropped a gigantic multimodal model called NVLM 72B 🦖  
+	Explaining everything from what I got of reading the paper here 📝
+        """,
+    'tweet_2':
+        """
+	The paper contains many ablation studies on various ways to use the LLM backbone 👇🏻  
+	
+	🦩 Flamingo-like cross-attention (NVLM-X)  
+	🌋 Llava-like concatenation of image and text embeddings to a decoder-only model (NVLM-D)  
+	✨ a hybrid architecture (NVLM-H)
+        """,
+    'tweet_3':
+        """
+	Checking evaluations, NVLM-D and NVLM-H are best or second best compared to other models 👏  
+
+	The released model is NVLM-D based on Qwen-2 Instruct, aligned with InternViT-6B using a huge mixture of different datasets        
+	""",
+    'tweet_4':
+        """
+	You can easily use this model by loading it through 🤗 Transformers' AutoModel 😍
+	 """,
+    'ressources':
+        """
+        Ressources:  
+        [NVLM: Open Frontier-Class Multimodal LLMs](https://arxiv.org/abs/2409.11402) 
+        by Wenliang Dai, Nayeon Lee, Boxin Wang, Zhuoling Yang, Zihan Liu, Jon Barker, Tuomas Rintamaki, Mohammad Shoeybi, Bryan Catanzaro, Wei Ping (2024)  
+        [GitHub](https://nvlm-project.github.io/)  
+        [Model](https://huggingface.co/nvidia/NVLM-D-72B)
+        """
+      },
+'fr': {
+    'title': 'NVLM',
+    'original_tweet': 
+       """
+       [Tweet de base](https://x.com/mervenoyann/status/1841098941900767323) (en anglais) (1er ocotbre 2024)
+       """,
+    'tweet_1':
+        """
+        NVIDIA vient de publier un gigantesque modèle multimodal appelé NVLM 72B 🦖  
+	J'explique tout ce que j'ai compris suite à la lecture du papier 📝
+        """,
+    'tweet_2':
+        """
+	L'article contient de nombreuses études d'ablation sur les différentes façons d'utiliser le backbone 👇🏻  
+	
+	🦩 Attention croisée de type Flamingo (NVLM-X)  
+	🌋 concaténation de type Llava d'embeddings d'images et de textes à un décodeur (NVLM-D)  
+	✨ une architecture hybride (NVLM-H)
+        """,
+    'tweet_3':
+        """
+	En vérifiant les évaluations, NVLM-D et NVLM-H sont les meilleurs ou les deuxièmes par rapport aux autres modèles 👏  
+
+	Le modèle publié est NVLM-D basé sur Qwen-2 Instruct, aligné avec InternViT-6B en utilisant un énorme mélange de différents jeux de données.        
+	""",
+    'tweet_4':
+        """
+	Vous pouvez facilement utiliser ce modèle en le chargeant via AutoModel de 🤗 Transformers 😍
+	 """,
+    'ressources':
+        """
+        Ressources :    
+        [NVLM: Open Frontier-Class Multimodal LLMs](https://arxiv.org/abs/2409.11402) 
+        de Wenliang Dai, Nayeon Lee, Boxin Wang, Zhuoling Yang, Zihan Liu, Jon Barker, Tuomas Rintamaki, Mohammad Shoeybi, Bryan Catanzaro, Wei Ping (2024)  
+        [GitHub](https://nvlm-project.github.io/)  
+        [Modèle](https://huggingface.co/nvidia/NVLM-D-72B)
+        """
+    }
+}    
+
+
+def language_selector():
+    languages = {'EN': '🇬🇧', 'FR': '🇫🇷'}
+    selected_lang = st.selectbox('', options=list(languages.keys()), format_func=lambda x: languages[x], key='lang_selector')
+    return 'en' if selected_lang == 'EN' else 'fr'
+
+left_column, right_column = st.columns([5, 1])
+
+# Add a selector to the right column
+with right_column:
+    lang = language_selector()
+
+# Add a title to the left column
+with left_column:
+    st.title(translations[lang]["title"])
+    
+st.success(translations[lang]["original_tweet"], icon="ℹ️")
+st.markdown(""" """)
+
+st.markdown(translations[lang]["tweet_1"], unsafe_allow_html=True)
+st.markdown(""" """)
+
+st.image("pages/NVLM/image_1.png", use_column_width=True)
+st.markdown(""" """)
+
+st.markdown(translations[lang]["tweet_2"], unsafe_allow_html=True)
+st.markdown(""" """)
+
+st.image("pages/NVLM/image_2.png", use_column_width=True)
+st.markdown(""" """)
+
+st.markdown(translations[lang]["tweet_3"], unsafe_allow_html=True)
+st.markdown(""" """)
+
+st.image("pages/NVLM/image_3.png", use_column_width=True)
+st.markdown(""" """)
+
+st.markdown(translations[lang]["tweet_4"], unsafe_allow_html=True)
+st.markdown(""" """)
+
+st.image("pages/NVLM/image_4.png", use_column_width=True)
+st.markdown(""" """)
+
+with st.expander ("Code"):
+    st.code("""
+	import torch 
+	from transformers import AutoModel 
+
+	path = "nvidia/NVLM-D-72B" 
+
+	model = AutoModel.from_pretrained( 
+		path, 
+		torch_dtype=torch.bfloat16, 
+		low_cpu_mem_usage=True, 
+		use_flash_attn=False, 
+		trust_remote_code=True).eval()
+    """)
+
+st.info(translations[lang]["ressources"], icon="📚")  
+
+st.markdown(""" """)
+st.markdown(""" """)
+st.markdown(""" """)
+col1, col2, col3= st.columns(3)
+with col1:
+    if lang == "en":
+        if st.button('Previous paper', use_container_width=True):
+            switch_page("NVEagle")
+    else:
+        if st.button('Papier précédent', use_container_width=True):
+            switch_page("NVEagle")
+with col2:
+    if lang == "en":
+        if st.button("Home", use_container_width=True):
+            switch_page("Home")
+    else:
+        if st.button("Accueil", use_container_width=True):
+            switch_page("Home")
+with col3:
+    if lang == "en":
+        if st.button("Next paper", use_container_width=True):
+            switch_page("Home")
+    else:
+        if st.button("Papier suivant", use_container_width=True):
+            switch_page("Home")