Initialise app with basic functionality

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,3 @@
+*.pyc
+*.ipynb
+*.csv

Dockerfile

@@ -0,0 +1,30 @@
+FROM python:3.10
+
+WORKDIR /src
+
+COPY requirements.txt .
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Set up a new user named "user" with user ID 1000
+RUN useradd -m -u 1000 user
+# Switch to the "user" user
+USER user
+# Set home to the user's home directory
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH \
+    PYTHONPATH=$HOME/app \
+	PYTHONUNBUFFERED=1 \
+	GRADIO_ALLOW_FLAGGING=never \
+	GRADIO_NUM_PORTS=1 \
+	GRADIO_SERVER_NAME=0.0.0.0 \
+	GRADIO_THEME=huggingface \
+	SYSTEM=spaces
+ 
+# Set the working directory to the user's home directory
+WORKDIR $HOME/app
+
+# Copy the current directory contents into the container at $HOME/app setting the owner to the user
+COPY --chown=user . $HOME/app
+
+CMD ["python", "app.py"]

app.py

@@ -0,0 +1,239 @@
+import gradio as gr
+from datetime import datetime
+import pandas as pd
+from transformers import pipeline
+# # Load in packages
+
+# +
+import os
+
+# Need to overwrite version of gradio present in Huggingface spaces as it doesn't have like buttons/avatars (Oct 2023)
+#os.system("pip uninstall -y gradio")
+os.system("pip install gradio==3.50.0")
+
+from typing import TypeVar
+#from langchain.embeddings import HuggingFaceEmbeddings#, HuggingFaceInstructEmbeddings
+#from langchain.vectorstores import FAISS
+import gradio as gr
+
+from transformers import AutoTokenizer
+
+# Alternative model sources
+import ctransformers
+
+PandasDataFrame = TypeVar('pd.core.frame.DataFrame')
+
+import chatfuncs.chatfuncs as chatf
+
+# Disable cuda devices if necessary
+#os.environ['CUDA_VISIBLE_DEVICES'] = '-1' 
+
+def create_hf_model(model_name):
+
+    tokenizer = AutoTokenizer.from_pretrained(model_name, model_max_length = chatf.context_length)
+
+    summariser = pipeline("summarization", model=model_name, tokenizer=tokenizer) # philschmid/bart-large-cnn-samsum
+
+    #from transformers import AutoModelForSeq2SeqLM,  AutoModelForCausalLM
+    
+    #     if torch_device == "cuda":
+    #         if "flan" in model_name:
+    #             model = AutoModelForSeq2SeqLM.from_pretrained(model_name, device_map="auto")
+    #         else:
+    #             model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto")
+    #     else:
+    #         if "flan" in model_name:
+    #             model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
+    #         else: 
+    #             model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True)
+
+    
+
+    return summariser, tokenizer, model_name
+
+def load_model(model_type, gpu_layers, gpu_config=None, cpu_config=None, torch_device=None):
+    print("Loading model ", model_type)
+
+    # Default values inside the function
+    if gpu_config is None:
+        gpu_config = chatf.gpu_config
+    if cpu_config is None:
+        cpu_config = chatf.cpu_config
+    if torch_device is None:
+        torch_device = chatf.torch_device
+
+    if model_type == "Mistral Open Orca (larger, slow)":
+        hf_checkpoint = 'TheBloke/MistralLite-7B-GGUF'
+
+        if torch_device == "cuda":
+            gpu_config.update_gpu(gpu_layers)
+        else:
+            gpu_config.update_gpu(gpu_layers)
+            cpu_config.update_gpu(gpu_layers)
+
+        print("Loading with", cpu_config.gpu_layers, "model layers sent to GPU.")
+
+        print(vars(gpu_config))
+        print(vars(cpu_config))
+
+        #try:
+            #model = ctransformers.AutoModelForCausalLM.from_pretrained('Aryanne/Orca-Mini-3B-gguf', model_type='llama', model_file='q5_0-orca-mini-3b.gguf', **vars(gpu_config)) # **asdict(CtransRunConfig_cpu())
+            #model = ctransformers.AutoModelForCausalLM.from_pretrained('Aryanne/Wizard-Orca-3B-gguf', model_type='llama', model_file='q4_1-wizard-orca-3b.gguf', **vars(gpu_config)) # **asdict(CtransRunConfig_cpu())
+            #model = ctransformers.AutoModelForCausalLM.from_pretrained('TheBloke/Mistral-7B-OpenOrca-GGUF', model_type='mistral', model_file='mistral-7b-openorca.Q4_K_M.gguf', **vars(gpu_config), hf=True) # **asdict(CtransRunConfig_cpu())
+            
+        #except:
+            #model = ctransformers.AutoModelForCausalLM.from_pretrained('Aryanne/Orca-Mini-3B-gguf', model_type='llama', model_file='q5_0-orca-mini-3b.gguf', **vars(cpu_config)) #**asdict(CtransRunConfig_gpu())
+            #model = ctransformers.AutoModelForCausalLM.from_pretrained('Aryanne/Wizard-Orca-3B-gguf', model_type='llama', model_file='q4_1-wizard-orca-3b.gguf', **vars(cpu_config)) # **asdict(CtransRunConfig_cpu())
+            #model = ctransformers.AutoModelForCausalLM.from_pretrained('TheBloke/Mistral-7B-OpenOrca-GGUF', model_type='mistral', model_file='mistral-7b-openorca.Q4_K_M.gguf', **vars(cpu_config), hf=True) # **asdict(CtransRunConfig_cpu())
+            
+        #tokenizer = ctransformers.AutoTokenizer.from_pretrained(model)
+        #summariser = pipeline("text-generation", model=model, tokenizer=tokenizer)
+
+        model = []
+        tokenizer = []
+        summariser = []
+
+    if model_type == "flan-t5-large-stacked-samsum":
+        # Huggingface chat model
+        hf_checkpoint = 'stacked-summaries/flan-t5-large-stacked-samsum-1024'#'declare-lab/flan-alpaca-base' # # #
+
+        summariser, tokenizer, model_type = create_hf_model(model_name = hf_checkpoint)
+
+    if model_type == "flan-t5-small-stacked-samsum":
+        # Huggingface chat model
+        hf_checkpoint = 'stacked-summaries/flan-t5-small-stacked-samsum-1024' #'philschmid/flan-t5-small-stacked-samsum'#'declare-lab/flan-alpaca-base' # # #
+
+
+        summariser, tokenizer, model_type = create_hf_model(model_name = hf_checkpoint)
+
+    chatf.model = summariser
+    chatf.tokenizer = tokenizer
+    chatf.model_type = model_type
+
+    load_confirmation = "Finished loading model: " + model_type
+
+    print(load_confirmation)
+    return model_type, load_confirmation, model_type
+
+# Both models are loaded on app initialisation so that users don't have to wait for the models to be downloaded
+#model_type = "Mistral Open Orca (larger, slow)"
+#load_model(model_type, chatf.gpu_layers, chatf.gpu_config, chatf.cpu_config, chatf.torch_device)
+
+model_type = "flan-t5-large-stacked-samsum"
+load_model(model_type, chatf.gpu_layers, chatf.gpu_config, chatf.cpu_config, chatf.torch_device)
+
+model_type = "flan-t5-small-stacked-samsum"
+load_model(model_type, 0, chatf.gpu_config, chatf.cpu_config, chatf.torch_device)
+
+today = datetime.now().strftime("%d%m%Y")
+today_rev = datetime.now().strftime("%Y%m%d")
+
+def summarise_text(text, text_df, length_slider, in_colnames, model_type):      
+         
+        if text_df == None:
+            in_colnames="text"
+            in_colnames_list_first = in_colnames
+
+            in_text_df = pd.DataFrame({in_colnames_list_first:[text]})
+            
+        else: 
+            in_text_df = pd.read_csv(text_df.name, delimiter = ",", low_memory=False, encoding='cp1252')
+            in_colnames_list_first = in_colnames.tolist()[0][0]
+
+        if model_type != "Mistral Open Orca (larger, slow)":
+            summarised_text = chatf.model(list(in_text_df[in_colnames_list_first]), max_length=length_slider)
+
+        if model_type == "Mistral Open Orca (larger, slow)":
+
+            length = str(length_slider)
+
+            prompt = """<|im_start|>system
+You are an AI assistant that follows instruction extremely well. Help as much as you can.
+<|im_start|>user
+Summarise the following text in less than {length} words.
+Text: {text}
+Answer:<|im_end|>"""
+
+            formatted_string = prompt.format(length=length, text=text)
+
+            print(formatted_string)
+
+            #summarised_text = chatf.model(formatted_string, max_new_tokens=length_slider)
+
+            summarised_text = "Mistral Open Orca summaries currently not working. Sorry!"
+
+        if text_df == None:
+            if model_type != "Mistral Open Orca (larger, slow)":
+                summarised_text_out = summarised_text[0].values()
+
+            if model_type == "Mistral Open Orca (larger, slow)":
+                summarised_text_out = summarised_text
+
+        else: 
+            summarised_text_out = [d['summary_text'] for d in summarised_text] #summarised_text[0].values()
+
+        output_name = "summarise_output_" + today_rev + ".csv"
+        output_df = pd.DataFrame({"Original text":in_text_df[in_colnames_list_first],
+                                  "Summarised text":summarised_text_out})
+        
+        summarised_text_out_str = str(output_df["Summarised text"][0])#.str.replace("dict_values([","").str.replace("])",""))
+        
+        output_df.to_csv(output_name, index = None)
+        
+        return summarised_text_out_str, output_name
+
+# ## Gradio app - summarise
+block = gr.Blocks(theme = gr.themes.Base())
+
+with block:  
+
+    model_type_state = gr.State(model_type)
+      
+    gr.Markdown(
+    """
+    # Text summariser
+    Enter open text below to get a summary. You can copy and paste text directly, or upload a file and specify the column that you want to summarise. Note that summarisation with Mistral Open Orca is still in development and does not currently work.
+    """)    
+    
+    with gr.Tab("Summariser"):
+        current_model = gr.Textbox(label="Current model", value=model_type, scale = 3)
+
+        with gr.Accordion("Paste open text", open = False):
+            in_text = gr.Textbox(label="Copy and paste your open text here", lines = 5)
+            
+        with gr.Accordion("Summarise open text from a file", open = False):
+            in_text_df = gr.File(label="Input text from file")
+            in_colnames = gr.Dataframe(label="Write the column name for the open text to summarise",
+                                    type="numpy", row_count=(1,"fixed"), col_count = (1,"fixed"),
+                                headers=["Open text column name"])#, "Address column name 2", "Address column name 3", "Address column name 4"])
+    
+        with gr.Row():
+            summarise_btn = gr.Button("Summarise")
+            length_slider = gr.Slider(minimum = 30, maximum = 200, value = 100, step = 10, label = "Maximum length of summary")
+        
+        with gr.Row():
+            output_single_text = gr.Textbox(label="Output example (first example in dataset)")
+            output_file = gr.File(label="Output file")
+
+    with gr.Tab("Advanced features"):
+        #out_passages = gr.Slider(minimum=1, value = 2, maximum=10, step=1, label="Choose number of passages to retrieve from the document. Numbers greater than 2 may lead to increased hallucinations or input text being truncated.")
+        #temp_slide = gr.Slider(minimum=0.1, value = 0.1, maximum=1, step=0.1, label="Choose temperature setting for response generation.")
+        with gr.Row():
+            model_choice = gr.Radio(label="Choose a summariser model", value="flan-t5-small-stacked-samsum", choices = ["flan-t5-small-stacked-samsum", "flan-t5-large-stacked-samsum", "Mistral Open Orca (larger, slow)"])
+            change_model_button = gr.Button(value="Load model", scale=0)
+        with gr.Accordion("Choose number of model layers to send to GPU (WARNING: please don't modify unless you are sure you have a GPU).", open = False):
+            gpu_layer_choice = gr.Slider(label="Choose number of model layers to send to GPU.", value=0, minimum=0, maximum=5, step = 1, visible=True)
+
+        load_text = gr.Text(label="Load status")
+
+
+    change_model_button.click(fn=load_model, inputs=[model_choice, gpu_layer_choice], outputs = [model_type_state, load_text, current_model])
+
+    summarise_btn.click(fn=summarise_text, inputs=[in_text, in_text_df, length_slider, in_colnames, model_type_state],
+                        outputs=[output_single_text, output_file], api_name="summarise_single_text")
+
+block.queue(concurrency_count=1).launch()
+# -
+
+
+

chatfuncs/chatfuncs.py

@@ -0,0 +1,112 @@
+
+from typing import TypeVar
+
+# Model packages
+import torch.cuda
+from transformers import pipeline
+
+torch.cuda.empty_cache()
+
+PandasDataFrame = TypeVar('pd.core.frame.DataFrame')
+
+model_type = None # global variable setup
+
+full_text = "" # Define dummy source text (full text) just to enable highlight function to load
+
+model = [] # Define empty list for model functions to run
+tokenizer = [] # Define empty list for model functions to run
+
+
+# Currently set gpu_layers to 0 even with cuda due to persistent bugs in implementation with cuda
+if torch.cuda.is_available():
+    torch_device = "cuda"
+    gpu_layers = 0
+else: 
+    torch_device =  "cpu"
+    gpu_layers = 0
+
+print("Running on device:", torch_device)
+threads = 8 #torch.get_num_threads()
+print("CPU threads:", threads)
+
+# flan-t5-large-stacked-xsum Model parameters
+temperature: float = 0.1
+top_k: int = 3
+top_p: float = 1
+repetition_penalty: float = 1.3
+flan_alpaca_repetition_penalty: float = 1.3
+last_n_tokens: int = 64
+max_new_tokens: int = 256
+seed: int = 42
+reset: bool = False
+stream: bool = True
+threads: int = threads
+batch_size:int = 256
+context_length:int = 4096
+sample = True
+
+
+class CtransInitConfig_gpu:
+    def __init__(self, temperature=temperature,
+                 top_k=top_k,
+                 top_p=top_p,
+                 repetition_penalty=repetition_penalty,
+                 last_n_tokens=last_n_tokens,
+                 max_new_tokens=max_new_tokens,
+                 seed=seed,
+                 reset=reset,
+                 stream=stream,
+                 threads=threads,
+                 batch_size=batch_size,
+                 context_length=context_length,
+                 gpu_layers=gpu_layers):
+        self.temperature = temperature
+        self.top_k = top_k
+        self.top_p = top_p
+        self.repetition_penalty = repetition_penalty# repetition_penalty
+        self.last_n_tokens = last_n_tokens
+        self.max_new_tokens = max_new_tokens
+        self.seed = seed
+        self.reset = reset
+        self.stream = stream
+        self.threads = threads
+        self.batch_size = batch_size
+        self.context_length = context_length
+        self.gpu_layers = gpu_layers
+        # self.stop: list[str] = field(default_factory=lambda: [stop_string])
+
+    def update_gpu(self, new_value):
+        self.gpu_layers = new_value
+
+class CtransInitConfig_cpu(CtransInitConfig_gpu):
+    def __init__(self):
+        super().__init__()
+        self.gpu_layers = 0
+
+gpu_config = CtransInitConfig_gpu()
+cpu_config = CtransInitConfig_cpu()
+
+
+class CtransGenGenerationConfig:
+    def __init__(self, temperature=temperature,
+                 top_k=top_k,
+                 top_p=top_p,
+                 repetition_penalty=repetition_penalty,
+                 last_n_tokens=last_n_tokens,
+                 seed=seed,
+                 threads=threads,
+                 batch_size=batch_size,
+                 reset=True
+                 ):
+        self.temperature = temperature
+        self.top_k = top_k
+        self.top_p = top_p
+        self.repetition_penalty = repetition_penalty# repetition_penalty
+        self.last_n_tokens = last_n_tokens
+        self.seed = seed
+        self.threads = threads
+        self.batch_size = batch_size
+        self.reset = reset
+
+    def update_temp(self, new_value):
+        self.temperature = new_value

readme.md

@@ -1 +1,13 @@
-test
+---
+title: Open Text Summariser
+emoji: 🚀
+colorFrom: green
+colorTo: gray
+sdk: gradio
+sdk_version: 3.50.0
+app_file: app.py
+pinned: false
+license: apache-2.0
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

requirements.txt

@@ -0,0 +1,4 @@
+gradio==3.50.0
+transformers
+torch
+ctransformers[cuda]