Rename app to rosetta. Make two-column. Add some texts.

README.md

@@ -1,6 +1,6 @@
 ---
-title: Babel - translate between Dutch and English
-emoji: 🧑‍💻
+title: Rosetta - translate between Dutch and English
+emoji: ‍🧑💻📑
 colorFrom: gray
 colorTo: granola
 sdk: streamlit

app.py

@@ -1,12 +1,12 @@
 import time
-import torch
 
 import psutil
 import streamlit as st
-
-from generator import GeneratorFactory
+import torch
 from langdetect import detect
+
 from default_texts import default_texts
+from generator import GeneratorFactory
 
 device = torch.cuda.device_count() - 1
 
@@ -32,9 +32,33 @@ GENERATOR_LIST = [
         "task": TRANSLATION_EN_TO_NL,
         "split_sentences": True,
     },
+    {
+        "model_name": "Helsinki-NLP/opus-mt-nl-en",
+        "desc": "Opus MT nl->en",
+        "task": TRANSLATION_NL_TO_EN,
+        "split_sentences": True,
+    },
+    # {
+    #     "model_name": "yhavinga/longt5-local-eff-large-nl8-voc8k-ddwn-512beta-512l-nedd-256ccmatrix-nl-en",
+    #     "desc": "longT5 large nl8 256cc/512beta/512l nl->en",
+    #     "task": TRANSLATION_NL_TO_EN,
+    #     "split_sentences": False,
+    # },
+    {
+        "model_name": "yhavinga/longt5-local-eff-large-nl8-voc8k-ddwn-512beta-512-nedd-nl-en",
+        "desc": "longT5 large nl8 512beta/512l nl->en",
+        "task": TRANSLATION_NL_TO_EN,
+        "split_sentences": False,
+    },
+    {
+        "model_name": "yhavinga/longt5-local-eff-large-nl8-voc8k-ddwn-512beta-512l-nedd-256ccmatrix-en-nl",
+        "desc": "longT5 large nl8 256cc/512beta/512l en->nl",
+        "task": TRANSLATION_EN_TO_NL,
+        "split_sentences": False,
+    },
     # {
-    #     "model_name": "yhavinga/longt5-local-eff-large-nl8-voc8k-ddwn-512beta-512l-nedd-256ccmatrix-en-nl",
-    #     "desc": "longT5 large nl8 256cc/512beta/512l en->nl",
+    #     "model_name": "yhavinga/longt5-local-eff-base-nl36-voc8k-256l-472beta-256l-472beta-en-nl",
+    #     "desc": "longT5 large nl8 256l/472beta/256l/472beta en->nl",
     #     "task": TRANSLATION_EN_TO_NL,
     #     "split_sentences": False,
     # },
@@ -79,10 +103,10 @@ GENERATOR_LIST = [
 
 def main():
     st.set_page_config(  # Alternate names: setup_page, page, layout
-        page_title="Babel",  # String or None. Strings get appended with "• Streamlit".
+        page_title="Rosetta en/nl",  # String or None. Strings get appended with "• Streamlit".
         layout="wide",  # Can be "centered" or "wide". In the future also "dashboard", etc.
-        initial_sidebar_state="expanded",  # Can be "auto", "expanded", "collapsed"
-        page_icon="📚",  # String, anything supported by st.image, or None.
+        initial_sidebar_state="auto",  # Can be "auto", "expanded", "collapsed"
+        page_icon="📑",  # String, anything supported by st.image, or None.
     )
 
     if "generators" not in st.session_state:
@@ -91,24 +115,28 @@ def main():
 
     with open("style.css") as f:
         st.markdown(f"<style>{f.read()}</style>", unsafe_allow_html=True)
-    st.sidebar.image("babel.png", width=200)
+
+    st.sidebar.image("rosetta.png", width=200)
     st.sidebar.markdown(
-        """# Babel
+        """# Rosetta
     Vertaal van en naar Engels"""
     )
-    st.sidebar.title("Parameters:")
 
     default_text = st.sidebar.radio(
         "Change default text",
         tuple(default_texts.keys()),
         index=0,
     )
-
     if default_text or "prompt_box" not in st.session_state:
         st.session_state["prompt_box"] = default_texts[default_text]["text"]
 
-    text_area = st.text_area("Enter text", st.session_state.prompt_box, height=300)
+    # create a left and right column
+    left, right = st.columns(2)
+    text_area = left.text_area("Enter text", st.session_state.prompt_box, height=500)
     st.session_state["text"] = text_area
+
+    # Sidebar parameters
+    st.sidebar.title("Parameters:")
     num_beams = st.sidebar.number_input("Num beams", min_value=1, max_value=10, value=1)
     num_beam_groups = st.sidebar.number_input(
         "Num beam groups", min_value=1, max_value=10, value=1
@@ -116,13 +144,11 @@ def main():
     length_penalty = st.sidebar.number_input(
         "Length penalty", min_value=0.0, max_value=2.0, value=1.2, step=0.1
     )
-
     st.sidebar.markdown(
         """For an explanation of the parameters, head over to the [Huggingface blog post about text generation](https://huggingface.co/blog/how-to-generate)
 and the [Huggingface text generation interface doc](https://huggingface.co/transformers/main_classes/model.html?highlight=generate#transformers.generation_utils.GenerationMixin.generate).
 """
     )
-
     params = {
         "num_beams": num_beams,
         "num_beam_groups": num_beam_groups,
@@ -130,7 +156,7 @@ and the [Huggingface text generation interface doc](https://huggingface.co/trans
         "early_stopping": True,
     }
 
-    if st.button("Run"):
+    if left.button("Run"):
         memory = psutil.virtual_memory()
 
         language = detect(st.session_state.text)
@@ -139,16 +165,16 @@ and the [Huggingface text generation interface doc](https://huggingface.co/trans
         elif language == "nl":
             task = TRANSLATION_NL_TO_EN
         else:
-            st.error(f"Language {language} not supported")
+            left.error(f"Language {language} not supported")
             return
 
         # Num beam groups should be a divisor of num beams
         if num_beams % num_beam_groups != 0:
-            st.error("Num beams should be a multiple of num beam groups")
+            left.error("Num beams should be a multiple of num beam groups")
             return
 
         for generator in generators.filter(task=task):
-            st.markdown(f"🧮 **Model `{generator}`**")
+            right.markdown(f"🧮 **Model `{generator}`**")
             time_start = time.time()
             result, params_used = generator.generate(
                 text=st.session_state.text, **params
@@ -156,9 +182,9 @@ and the [Huggingface text generation interface doc](https://huggingface.co/trans
             time_end = time.time()
             time_diff = time_end - time_start
 
-            st.write(result.replace("\n", "  \n"))
+            right.write(result.replace("\n", "  \n"))
             text_line = ", ".join([f"{k}={v}" for k, v in params_used.items()])
-            st.markdown(f"    🕙 *generated in {time_diff:.2f}s, `{text_line}`*")
+            right.markdown(f"    🕙 *generated in {time_diff:.2f}s, `{text_line}`*")
 
         st.write(
             f"""

default_texts.py

@@ -10,14 +10,36 @@ It was a quite young girl, unknown to me, with a hood over her head, and with la
 
 “My father is very ill,” she said without a word of introduction. “The nurse is frightened. Could you come in and help?”""",
     },
-    "Gedachten": {
-        "url": "https://www.dbnl.org/tekst/eede003geda01_01/eede003geda01_01_0001.php",
-        "year": 1920,
-        "text": """Verdraagzaamheid en gewetensvrijheid brengen voor velen mee, dat ze noodwendig hun overtuiging een beetje onderdrukken, dat ze hun ware meening verzwijgen.
+    "ISS Crash 2031": {
+        "url": "https://www.bbc.com/news/science-environment-60246032",
+        "year": 2022,
+        "text": """The International Space Station (ISS) will continue working until 2030, before plunging into the Pacific Ocean in early 2031, according to Nasa.
 
-Ieder gezond, jong mensch voelt op een zekeren leeftijd dat zijn hoogste aspiraties, zijn beste capaciteiten worden neergehouden en doodgedrukt door wat men maatschappelijke verplichting noemt.
+In a report this week, the US space agency said the ISS would crash into a part of the ocean known as Point Nemo.
 
-Drijf uw kinderen niet met dweepzucht of bekrompenheid in dezen of genen afgepaalden weg, maar geef hen ruimte, zoek voor hun geest alle voedsel, dat zij van nature behoeven en waarbij zij gedijen en vreugde vinden.
-""",
+This is the point furthest from land on planet Earth, also known as the spacecraft cemetery.
+
+Many old satellites and other space debris have crashed there, including the Russian space station Mir in 2001.
+
+Nasa said that in the future space activities close to Earth would be led by the commercial sector.
+
+The ISS - a joint project involving five space agencies - has been in orbit since 1998 and has been continuously crewed since 2000. More than 3,000 research investigations have taken place in its microgravity laboratory.""",
+    },
+    "Fish Dinner": {
+        "url": "https://www.goodreads.com/en/book/show/274065",
+        "year": 1941,
+        "text": """You must consider of it less narrowly: sub specie aeternitatis. Supposition is, every conceivable bunch of circumstances, that is to say, every conceivable world, exists: but unworlded, unbunched: to our more mean capacities an unpassable bog or flux of seas, cities, rivers, lakes, wolds and deserts and mountain ranges, all with their plants, forests, mosses, water-weeds, what you will; and all manner of peoples, beasts, birds, fishes, creeping things, climes, dreams, loves, loathings, abominations, ecstasies, dissolutions, hopes, fears, forgetfulnesses, infinite in variety, infinite in number, fantasies beyond nightmare or madness. All this in potentia. All are there, even-just as are all the particulars in a landscape: He, like as the landscape-painter, selects and orders. The one paints a picture, the Other creates a world.""",
+    },
+    "Hersenschimmen": {
+        "url": "https://www.bibliotheek.nl/catalogus/titel.37120397X.html/hersenschimmen/",
+        "year": 1960,
+        "text": """Misschien komt het door de sneeuw dat ik me ’s morgens al zo moe voel. Vera niet, zij houdt van sneeuw. Volgens haar gaat er niks boven een sneeuwlandschap. Als de sporen van de mens uit de natuur verdwijnen, als alles één smetteloze witte vlakte wordt; zo mooi! Dwepend bijna zegt ze dat. Maar lang duurt die toestand hier niet. Al na een paar uur zie je overal schoenafdrukken, bandensporen en worden de hoofdwegen door sneeuwruimers schoongeploegd.
+Ik hoor haar in de keuken bezig met de koffie. Alleen de okergele haltepaal van de schoolbus geeft nog aan waar de Field Road langs ons huis loopt. Ik begrijp trouwens niet waar de kinderen blijven vandaag. Iedere ochtend sta ik hier zo voor het raam. Eerst controleer ik de temperatuur en dan wacht ik tot ze in de vroege winterochtend van alle kanten tussen de boomstammen tevoorschijn komen met hun rugtassen, hun kleurige mutsen en dassen en hun schelle Amerikaanse stemmen. Die bonte kleuren stemmen me vrolijk. Vuurrood, kobaltblauw. Eén jongetje heeft een eigeel jack aan met een pauw op de rug geborduurd, een jongetje dat licht hinkt en altijd als laatste in de schoolbus klautert. Dat is Richard, de zoon van Tom, de vuurtorenwachter, geboren met een te kort linkerbeen. Een hemelsblauw wijduitstaande pauwenstaart vol donker starende ogen. Ik begrijp niet waar ze blijven vandaag.""",
+    },
+    "De Uitvreter": {
+        "url": "https://www.jeugdbibliotheek.nl/12-18-jaar/lezen-voor-de-lijst/15-18-jaar/niveau-5/de-uitvreter-titaantjes-dichtertje.html",
+        "year": 1911,
+        "text": """‘Is u Amsterdammer?’ vroeg Bavink. ‘Ja, Goddank,’ zei Japi. ‘Ik ook,’ zei Bavink. ‘U schildert niet?’ vroeg Bavink. Het was een rare burgermansvraag, maar Bavink dacht aldoor maar: wat zou dat toch voor een kerel wezen? ‘Nee, Goddank,’ zei Japi, ‘en ik dicht ook niet en ik ben geen natuurvriend en geen anarchist. Ik ben Goddank heelemaal niks.’
+Dat kon Bavink wel bekoren.""",
     },
 }

generator.py

@@ -3,10 +3,7 @@ import re
 
 import streamlit as st
 import torch
-from transformers import (
-    AutoModelForSeq2SeqLM,
-    AutoTokenizer,
-)
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
 
 device = torch.cuda.device_count() - 1
 

requirements.txt

@@ -11,4 +11,4 @@ jax[cuda]==0.3.16
 chex>=0.1.4
 ##jaxlib==0.1.67
 flax>=0.5.3
-sentencepiece
+sentencepiece