Duplicate from Yusin/Speech2ChatGPT2Speech

Co-authored-by: Chen, Yusin <[email protected]>

.gitattributes

@@ -0,0 +1,27 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+.env

README.md

@@ -0,0 +1,41 @@
+---
+title: Speech2ChatGPT2Speech
+emoji: 🗣️🙉
+colorFrom: green
+colorTo: green
+sdk: gradio
+python_version: 3.9
+sdk_version: 3.2
+app_file: app.py
+models:
+- neongeckocom/tts-vits-ljspeech-en
+- neongeckocom/tts-vits-css10-es
+- neongeckocom/tts-vits-css10-fr
+- neongeckocom/tts-vits-css10-de
+- neongeckocom/tts-vits-cv-it
+- neongeckocom/tts-vits-mai-pl
+- neongeckocom/tts-vits-mai-uk
+- neongeckocom/tts-vits-cv-ro
+- neongeckocom/tts-vits-css10-hu
+- neongeckocom/tts-vits-cv-el
+- neongeckocom/tts-vits-cv-cs
+- neongeckocom/tts-vits-cv-sv
+- neongeckocom/tts-vits-cv-pt
+- neongeckocom/tts-vits-cv-bg
+- neongeckocom/tts-vits-cv-hr
+- neongeckocom/tts-vits-cv-da
+- neongeckocom/tts-vits-cv-sk
+- neongeckocom/tts-vits-css10-nl
+- neongeckocom/tts-vits-css10-fi
+- neongeckocom/tts-vits-cv-lt
+- neongeckocom/tts-vits-cv-sl
+- neongeckocom/tts-vits-cv-lv
+- neongeckocom/tts-vits-cv-et
+- neongeckocom/tts-vits-cv-ga
+- neongeckocom/tts-vits-cv-mt
+pinned: false
+license: bsd-3-clause
+duplicated_from: Yusin/Speech2ChatGPT2Speech
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces#reference

app.py

@@ -0,0 +1,106 @@
+import tempfile
+import gradio as gr
+from neon_tts_plugin_coqui import CoquiTTS
+LANGUAGES = list(CoquiTTS.langs.keys())
+default_lang = "en"
+
+# ChatGPT
+from pyChatGPT import ChatGPT
+import whisper
+whisper_model = whisper.load_model("small")
+import os
+session_token = os.environ.get('SessionToken') 
+
+title = "Speech to ChatGPT to Speech"
+#info = "more info at [Neon Coqui TTS Plugin](https://github.com/NeonGeckoCom/neon-tts-plugin-coqui), [Coqui TTS](https://github.com/coqui-ai/TTS)"
+#badge = "https://visitor-badge-reloaded.herokuapp.com/badge?page_id=neongeckocom.neon-tts-plugin-coqui"
+
+coquiTTS = CoquiTTS()
+
+
+# ChatGPT
+def chat_hf(audio, custom_token, language):
+    try:
+        whisper_text = translate(audio)
+        api = ChatGPT(session_token) 
+        resp = api.send_message(whisper_text)
+        
+        api.refresh_auth()  # refresh the authorization token
+        api.reset_conversation()  # reset the conversation
+        gpt_response = resp['message']
+
+    except:
+        whisper_text = translate(audio)
+        api = ChatGPT(custom_token) 
+        resp = api.send_message(whisper_text)
+        
+        api.refresh_auth()  # refresh the authorization token
+        api.reset_conversation()  # reset the conversation
+        gpt_response = resp['message']
+
+    # to voice
+    with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as fp:
+        coquiTTS.get_tts(gpt_response, fp, speaker = {"language" : language})
+    
+    return whisper_text, gpt_response, fp.name
+
+# whisper
+def translate(audio):
+    print("""
+    —
+    Sending audio to Whisper ...
+    —
+    """)
+    
+    audio = whisper.load_audio(audio)
+    audio = whisper.pad_or_trim(audio)
+    
+    mel = whisper.log_mel_spectrogram(audio).to(whisper_model.device)
+    
+    _, probs = whisper_model.detect_language(mel)
+    
+    transcript_options = whisper.DecodingOptions(task="transcribe", fp16 = False)
+    
+    transcription = whisper.decode(whisper_model, mel, transcript_options)
+    
+    print("language spoken: " + transcription.language)
+    print("transcript: " + transcription.text)
+    print("———————————————————————————————————————————")  
+      
+    return transcription.text
+
+with gr.Blocks() as blocks:
+    gr.Markdown("<h1 style='text-align: center; margin-bottom: 1rem'>"
+                + title
+                + "</h1>")
+    #gr.Markdown(description)
+    with gr.Row(equal_height=True):# equal_height=False
+        with gr.Column():# variant="panel"
+            radio = gr.Radio(
+                label="Language",
+                choices=LANGUAGES,
+                value=default_lang
+            )
+            audio_file = gr.inputs.Audio(source="microphone", type="filepath")
+            custom_token = gr.Textbox(label='If it fails, use your own session token', placeholder="your own session token")
+            with gr.Row():# mobile_collapse=False
+                submit = gr.Button("Submit", variant="primary")
+        with gr.Column():
+            text1 = gr.Textbox(label="Speech to Text")
+            text2 = gr.Textbox(label="chatGPT response")
+            audio = gr.Audio(label="Output", interactive=False)
+    #gr.Markdown(info)
+    #gr.Markdown("<center>"
+    #            +f'<img src={badge} alt="visitors badge"/>'
+    #            +"</center>")
+
+    # actions
+    submit.click(
+        chat_hf,
+        [audio_file, custom_token, radio],
+        [text1, text2, audio],
+    )
+    radio.change(lambda lang: CoquiTTS.langs[lang]["sentence"], radio, text2)
+
+
+blocks.launch(debug=True)

packages.txt

@@ -0,0 +1,2 @@
+libsndfile1
+espeak-ng

requirements.txt

@@ -0,0 +1,3 @@
+neon-tts-plugin-coqui==0.6.0
+pyChatGPT
+git+https://github.com/openai/whisper.git