Update preset tokenizers

Delete prompt file if exists
Log system info

app.py

@@ -1,25 +1,28 @@
+import pprint
+import subprocess
+from pathlib import Path
+
 import gradio as gr
-from test_prompt_generator import generate_prompt
+from test_prompt_generator.test_prompt_generator import _preset_tokenizers, generate_prompt
+
+# log system info for debugging purposes
+result = subprocess.run(["lscpu"], text=True, capture_output=True)
+pprint.pprint(result.stdout)
+result = subprocess.run(["pip", "freeze"], text=True, capture_output=True)
+pprint.pprint(result.stdout)
 
-tokenizers = [
-    "google/gemma-7b",
-    "mistralai/Mistral-7B-v0.1",
-    "facebook/opt-2.7b",
-    "microsoft/phi-2",
-    "THUDM/chatglm3-6b",
-    "Qwen/Qwen1.5-7B-Chat",
-    "bigscience/bloom-560m",
-    "ise-uiuc/Magicoder-S-DS-6.7B",
-    "google/flan-t5-base",
-    "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
-    "google-bert/bert-base-uncased",
-]
 
+def generate(tokenizer_id, num_tokens, prefix=None, source_text=None):
+    output_path = Path(f"prompt_{num_tokens}.jsonl")
+    if output_path.exists():
+        output_path.unlink()
 
-def generate(model_id, num_tokens, prefix=None, source_text=None):
-    output_file = f"prompt_{num_tokens}.jsonl"
-    prompt = generate_prompt(model_id, int(num_tokens), prefix=prefix, source_text=source_text, output_file=output_file)
-    return prompt, output_file
+    prompt = generate_prompt(
+        tokenizer_id, int(num_tokens), prefix=prefix, source_text=source_text, output_file=output_path
+    )
+    if tokenizer_id in _preset_tokenizers:
+        tokenizer_id = _preset_tokenizers[tokenizer_id]
+    return prompt, str(output_path), tokenizer_id
 
 
 demo = gr.Interface(
@@ -30,8 +33,8 @@ demo = gr.Interface(
     inputs=[
         gr.Dropdown(
             label="Tokenizer",
-            choices=tokenizers,
-            value="mistralai/Mistral-7B-v0.1",
+            choices=_preset_tokenizers,
+            value="mistral",
             allow_custom_value=True,
             info="Select a tokenizer from this list or paste a model_id from a model on the Hugging Face Hub",
         ),
@@ -47,14 +50,18 @@ demo = gr.Interface(
             info="By default, prompts will be generated from Alice in Wonderland. Enter text here to use that instead.",
         ),
     ],
-    outputs=[gr.Textbox(label="prompt", show_copy_button=True), gr.File(label="Json file")],
+    outputs=[
+        gr.Textbox(label="prompt", show_copy_button=True),
+        gr.File(label="Json file"),
+        gr.Markdown(label="tokenizer"),
+    ],
     examples=[
-        ["mistralai/Mistral-7B-v0.1", 32],
-        ["mistralai/Mistral-7B-v0.1", 64],
-        ["mistralai/Mistral-7B-v0.1", 128],
-        ["mistralai/Mistral-7B-v0.1", 512],
-        ["mistralai/Mistral-7B-v0.1", 1024],
-        ["mistralai/Mistral-7B-v0.1", 2048],
+        ["mistral", 32],
+        ["mistral", 64],
+        ["mistral", 128],
+        ["mistral", 512],
+        ["mistral", 1024],
+        ["mistral", 2048],
     ],
     cache_examples=False,
     allow_flagging=False,