Spaces:
Runtime error
Runtime error
| import os | |
| import ast | |
| import time | |
| from enums import PromptType # also supports imports from this file from other files | |
| non_hf_types = ['gpt4all_llama', 'llama', 'gptj'] | |
| prompt_type_to_model_name = { | |
| 'plain': [ | |
| 'EleutherAI/gpt-j-6B', | |
| 'EleutherAI/pythia-6.9b', | |
| 'EleutherAI/pythia-12b', | |
| 'EleutherAI/pythia-12b-deduped', | |
| 'EleutherAI/gpt-neox-20b', | |
| 'openlm-research/open_llama_7b_700bt_preview', | |
| 'decapoda-research/llama-7b-hf', | |
| 'decapoda-research/llama-13b-hf', | |
| 'decapoda-research/llama-30b-hf', | |
| 'decapoda-research/llama-65b-hf', | |
| 'facebook/mbart-large-50-many-to-many-mmt', | |
| 'philschmid/bart-large-cnn-samsum', | |
| 'philschmid/flan-t5-base-samsum', | |
| 'gpt2', | |
| 'distilgpt2', | |
| 'mosaicml/mpt-7b-storywriter', | |
| ], | |
| 'gptj': ['gptj', 'gpt4all_llama'], | |
| 'prompt_answer': [ | |
| 'h2oai/h2ogpt-gm-oasst1-en-1024-20b', | |
| 'h2oai/h2ogpt-gm-oasst1-en-1024-12b', | |
| 'h2oai/h2ogpt-gm-oasst1-multilang-1024-20b', | |
| 'h2oai/h2ogpt-gm-oasst1-multilang-2048-falcon-7b', | |
| 'h2oai/h2ogpt-gm-oasst1-multilang-2048-falcon-7b-v2', | |
| 'h2oai/h2ogpt-gm-oasst1-en-2048-falcon-7b-v3', | |
| 'h2oai/h2ogpt-gm-oasst1-en-2048-falcon-7b', | |
| 'h2oai/h2ogpt-gm-oasst1-en-2048-falcon-7b-v2', | |
| 'h2oai/h2ogpt-gm-oasst1-en-2048-falcon-40b-v1', | |
| 'h2oai/h2ogpt-gm-oasst1-en-2048-falcon-40b-v2', | |
| 'h2oai/h2ogpt-gm-oasst1-en-xgen-7b-8k', | |
| 'h2oai/h2ogpt-gm-oasst1-multilang-xgen-7b-8k', | |
| 'TheBloke/h2ogpt-gm-oasst1-en-2048-falcon-40b-v2-GPTQ', | |
| ], | |
| 'prompt_answer_openllama': [ | |
| 'h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt', | |
| 'h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt-v2', | |
| 'h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-700bt', | |
| 'h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b', | |
| 'h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-13b', | |
| ], | |
| 'instruct': ['TheBloke/llama-30b-supercot-SuperHOT-8K-fp16'], # https://huggingface.co/TheBloke/llama-30b-supercot-SuperHOT-8K-fp16#prompting | |
| 'instruct_with_end': ['databricks/dolly-v2-12b'], | |
| 'quality': [], | |
| 'human_bot': [ | |
| 'h2oai/h2ogpt-oasst1-512-12b', | |
| 'h2oai/h2ogpt-oasst1-512-20b', | |
| 'h2oai/h2ogpt-oig-oasst1-256-6_9b', | |
| 'h2oai/h2ogpt-oig-oasst1-512-6_9b', | |
| 'h2oai/h2ogpt-oig-oasst1-256-6.9b', # legacy | |
| 'h2oai/h2ogpt-oig-oasst1-512-6.9b', # legacy | |
| 'h2oai/h2ogpt-research-oasst1-512-30b', | |
| 'h2oai/h2ogpt-research-oasst1-llama-65b', | |
| 'h2oai/h2ogpt-oasst1-falcon-40b', | |
| 'h2oai/h2ogpt-oig-oasst1-falcon-40b', | |
| ], | |
| 'dai_faq': [], | |
| 'summarize': [], | |
| 'simple_instruct': ['t5-small', 't5-large', 'google/flan-t5', 'google/flan-t5-xxl', 'google/flan-ul2'], | |
| 'instruct_vicuna': ['AlekseyKorshuk/vicuna-7b', 'TheBloke/stable-vicuna-13B-HF', 'junelee/wizard-vicuna-13b'], | |
| 'human_bot_orig': ['togethercomputer/GPT-NeoXT-Chat-Base-20B'], | |
| "open_assistant": ['OpenAssistant/oasst-sft-7-llama-30b-xor', 'oasst-sft-7-llama-30b'], | |
| "wizard_lm": ['ehartford/WizardLM-7B-Uncensored', 'ehartford/WizardLM-13B-Uncensored'], | |
| "wizard_mega": ['openaccess-ai-collective/wizard-mega-13b'], | |
| "instruct_simple": ['JosephusCheung/Guanaco'], | |
| "wizard_vicuna": ['ehartford/Wizard-Vicuna-13B-Uncensored'], | |
| "wizard2": ['llama'], | |
| "mptinstruct": ['mosaicml/mpt-30b-instruct', 'mosaicml/mpt-7b-instruct', 'mosaicml/mpt-30b-instruct'], | |
| "mptchat": ['mosaicml/mpt-7b-chat', 'mosaicml/mpt-30b-chat', 'TheBloke/mpt-30B-chat-GGML'], | |
| "vicuna11": ['lmsys/vicuna-33b-v1.3'], | |
| "falcon": ['tiiuae/falcon-40b-instruct', 'tiiuae/falcon-40b', 'tiiuae/falcon-7b-instruct', 'tiiuae/falcon-7b'], | |
| "llama2": [ | |
| 'meta-llama/Llama-2-7b-chat-hf', | |
| 'meta-llama/Llama-2-13b-chat-hf', | |
| 'meta-llama/Llama-2-34b-chat-hf', | |
| 'meta-llama/Llama-2-70b-chat-hf', | |
| ], | |
| # could be plain, but default is correct prompt_type for default TheBloke model ggml-wizardLM-7B.q4_2.bin | |
| } | |
| if os.getenv('OPENAI_API_KEY'): | |
| prompt_type_to_model_name.update({ | |
| "openai": ["text-davinci-003", "text-curie-001", "text-babbage-001", "text-ada-001"], | |
| "openai_chat": ["gpt-3.5-turbo", "gpt-3.5-turbo-16k"], | |
| }) | |
| inv_prompt_type_to_model_name = {v.strip(): k for k, l in prompt_type_to_model_name.items() for v in l} | |
| inv_prompt_type_to_model_lower = {v.strip().lower(): k for k, l in prompt_type_to_model_name.items() for v in l} | |
| prompt_types_strings = [] | |
| for p in PromptType: | |
| prompt_types_strings.extend([p.name]) | |
| prompt_types = [] | |
| for p in PromptType: | |
| prompt_types.extend([p.name, p.value, str(p.value)]) | |
| def get_prompt(prompt_type, prompt_dict, chat, context, reduced, making_context, return_dict=False): | |
| prompt_dict_error = '' | |
| generates_leading_space = False | |
| if prompt_type == PromptType.custom.name and not isinstance(prompt_dict, dict): | |
| try: | |
| prompt_dict = ast.literal_eval(prompt_dict) | |
| except BaseException as e: | |
| prompt_dict_error = str(e) | |
| if prompt_dict_error: | |
| promptA = None | |
| promptB = None | |
| PreInstruct = None | |
| PreInput = '' | |
| PreResponse = '' | |
| terminate_response = None | |
| chat_sep = '' | |
| chat_turn_sep = '' | |
| humanstr = '' | |
| botstr = '' | |
| generates_leading_space = False | |
| elif prompt_type in [PromptType.custom.value, str(PromptType.custom.value), | |
| PromptType.custom.name]: | |
| promptA = prompt_dict.get('promptA', '') | |
| promptB = prompt_dict.get('promptB', '') | |
| PreInstruct = prompt_dict.get('PreInstruct', '') | |
| PreInput = prompt_dict.get('PreInput', '') | |
| PreResponse = prompt_dict.get('PreResponse', '') | |
| terminate_response = prompt_dict.get('terminate_response', None) | |
| chat_sep = prompt_dict.get('chat_sep', '\n') | |
| chat_turn_sep = prompt_dict.get('chat_turn_sep', '\n') | |
| humanstr = prompt_dict.get('humanstr', '') | |
| botstr = prompt_dict.get('botstr', '') | |
| elif prompt_type in [PromptType.plain.value, str(PromptType.plain.value), | |
| PromptType.plain.name]: | |
| promptA = promptB = PreInstruct = PreInput = PreResponse = None | |
| terminate_response = [] | |
| chat_turn_sep = chat_sep = '' | |
| # plain should have None for human/bot, so nothing truncated out, not '' that would truncate after first token | |
| humanstr = None | |
| botstr = None | |
| elif prompt_type == 'simple_instruct': | |
| promptA = promptB = PreInstruct = PreInput = PreResponse = None | |
| terminate_response = [] | |
| chat_turn_sep = chat_sep = '\n' | |
| humanstr = None | |
| botstr = None | |
| elif prompt_type in [PromptType.instruct.value, str(PromptType.instruct.value), | |
| PromptType.instruct.name] + [PromptType.instruct_with_end.value, | |
| str(PromptType.instruct_with_end.value), | |
| PromptType.instruct_with_end.name]: | |
| promptA = 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n' if not ( | |
| chat and reduced) else '' | |
| promptB = 'Below is an instruction that describes a task. Write a response that appropriately completes the request.\n' if not ( | |
| chat and reduced) else '' | |
| PreInstruct = """ | |
| ### Instruction: | |
| """ | |
| PreInput = """ | |
| ### Input: | |
| """ | |
| PreResponse = """ | |
| ### Response: | |
| """ | |
| if prompt_type in [PromptType.instruct_with_end.value, str(PromptType.instruct_with_end.value), | |
| PromptType.instruct_with_end.name]: | |
| terminate_response = ['### End'] | |
| else: | |
| terminate_response = None | |
| chat_turn_sep = chat_sep = '\n' | |
| humanstr = PreInstruct | |
| botstr = PreResponse | |
| elif prompt_type in [PromptType.quality.value, str(PromptType.quality.value), | |
| PromptType.quality.name]: | |
| promptA = 'Write a detailed high-quality, accurate, fair, Response with about 100 words by following the Instruction as applied on the Input.\n' if not ( | |
| chat and reduced) else '' | |
| promptB = 'Write a detailed high-quality, accurate, fair, Response with about 100 words by following the Instruction.\n' if not ( | |
| chat and reduced) else '' | |
| PreInstruct = """ | |
| ### Instruction: | |
| """ | |
| PreInput = """ | |
| ### Input: | |
| """ | |
| PreResponse = """ | |
| ### Response: | |
| """ | |
| terminate_response = None | |
| chat_turn_sep = chat_sep = '\n' | |
| humanstr = PreInstruct # first thing human says | |
| botstr = PreResponse # first thing bot says | |
| elif prompt_type in [PromptType.human_bot.value, str(PromptType.human_bot.value), | |
| PromptType.human_bot.name] + [PromptType.human_bot_orig.value, | |
| str(PromptType.human_bot_orig.value), | |
| PromptType.human_bot_orig.name]: | |
| human = '<human>:' | |
| bot = "<bot>:" | |
| if reduced or context or prompt_type in [PromptType.human_bot.value, str(PromptType.human_bot.value), | |
| PromptType.human_bot.name]: | |
| preprompt = '' | |
| else: | |
| cur_date = time.strftime('%Y-%m-%d') | |
| cur_time = time.strftime('%H:%M:%S %p %Z') | |
| PRE_PROMPT = """\ | |
| Current Date: {} | |
| Current Time: {} | |
| """ | |
| preprompt = PRE_PROMPT.format(cur_date, cur_time) | |
| start = '' | |
| promptB = promptA = '%s%s' % (preprompt, start) | |
| PreInstruct = human + ' ' | |
| PreInput = None | |
| if making_context: | |
| # when making context, want it to appear as-if LLM generated, which starts with space after : | |
| PreResponse = bot + ' ' | |
| else: | |
| # normally LLM adds space after this, because was how trained. | |
| # if add space here, non-unique tokenization will often make LLM produce wrong output | |
| PreResponse = bot | |
| terminate_response = ['\n' + human, '\n' + bot, human, bot, PreResponse] | |
| chat_turn_sep = chat_sep = '\n' | |
| humanstr = human # tag before human talks | |
| botstr = bot # tag before bot talks | |
| generates_leading_space = True | |
| elif prompt_type in [PromptType.dai_faq.value, str(PromptType.dai_faq.value), | |
| PromptType.dai_faq.name]: | |
| promptA = '' | |
| promptB = 'Answer the following Driverless AI question.\n' | |
| PreInstruct = """ | |
| ### Driverless AI frequently asked question: | |
| """ | |
| PreInput = None | |
| PreResponse = """ | |
| ### Driverless AI documentation answer: | |
| """ | |
| terminate_response = ['\n\n'] | |
| chat_turn_sep = chat_sep = terminate_response | |
| humanstr = PreInstruct | |
| botstr = PreResponse | |
| elif prompt_type in [PromptType.summarize.value, str(PromptType.summarize.value), | |
| PromptType.summarize.name]: | |
| promptA = promptB = PreInput = '' | |
| PreInstruct = '## Main Text\n\n' | |
| PreResponse = '\n\n## Summary\n\n' | |
| terminate_response = None | |
| chat_turn_sep = chat_sep = '\n' | |
| humanstr = PreInstruct | |
| botstr = PreResponse | |
| elif prompt_type in [PromptType.instruct_vicuna.value, str(PromptType.instruct_vicuna.value), | |
| PromptType.instruct_vicuna.name]: | |
| promptA = promptB = "A chat between a curious human and an artificial intelligence assistant. " \ | |
| "The assistant gives helpful, detailed, and polite answers to the human's questions." if not ( | |
| chat and reduced) else '' | |
| PreInstruct = """ | |
| ### Human: | |
| """ | |
| PreInput = None | |
| PreResponse = """ | |
| ### Assistant: | |
| """ | |
| terminate_response = [ | |
| '### Human:'] # but only allow terminate after prompt is found correctly, else can't terminate | |
| chat_turn_sep = chat_sep = '\n' | |
| humanstr = PreInstruct | |
| botstr = PreResponse | |
| elif prompt_type in [PromptType.prompt_answer.value, str(PromptType.prompt_answer.value), | |
| PromptType.prompt_answer.name]: | |
| preprompt = '' | |
| prompt_tokens = "<|prompt|>" | |
| answer_tokens = "<|answer|>" | |
| start = '' | |
| promptB = promptA = '%s%s' % (preprompt, start) | |
| PreInstruct = prompt_tokens | |
| PreInput = None | |
| PreResponse = answer_tokens | |
| eos = '<|endoftext|>' # neox eos | |
| humanstr = prompt_tokens | |
| botstr = answer_tokens | |
| terminate_response = [humanstr, PreResponse, eos] | |
| chat_sep = eos | |
| chat_turn_sep = eos | |
| elif prompt_type in [PromptType.prompt_answer_openllama.value, str(PromptType.prompt_answer_openllama.value), | |
| PromptType.prompt_answer_openllama.name]: | |
| preprompt = '' | |
| prompt_tokens = "<|prompt|>" | |
| answer_tokens = "<|answer|>" | |
| start = '' | |
| promptB = promptA = '%s%s' % (preprompt, start) | |
| PreInstruct = prompt_tokens | |
| PreInput = None | |
| PreResponse = answer_tokens | |
| eos = '</s>' # llama eos | |
| humanstr = prompt_tokens | |
| botstr = answer_tokens | |
| terminate_response = [humanstr, PreResponse, eos] | |
| chat_sep = eos | |
| chat_turn_sep = eos | |
| elif prompt_type in [PromptType.open_assistant.value, str(PromptType.open_assistant.value), | |
| PromptType.open_assistant.name]: | |
| # From added_tokens.json | |
| preprompt = '' | |
| prompt_tokens = "<|prompter|>" | |
| answer_tokens = "<|assistant|>" | |
| start = '' | |
| promptB = promptA = '%s%s' % (preprompt, start) | |
| PreInstruct = prompt_tokens | |
| PreInput = None | |
| PreResponse = answer_tokens | |
| pend = "<|prefix_end|>" | |
| eos = "</s>" | |
| humanstr = prompt_tokens | |
| botstr = answer_tokens | |
| terminate_response = [humanstr, PreResponse, pend, eos] | |
| chat_turn_sep = chat_sep = eos | |
| elif prompt_type in [PromptType.wizard_lm.value, str(PromptType.wizard_lm.value), | |
| PromptType.wizard_lm.name]: | |
| # https://github.com/ehartford/WizardLM/blob/main/src/train_freeform.py | |
| preprompt = '' | |
| start = '' | |
| promptB = promptA = '%s%s' % (preprompt, start) | |
| PreInstruct = "" | |
| PreInput = None | |
| PreResponse = "\n\n### Response\n" | |
| eos = "</s>" | |
| terminate_response = [PreResponse, eos] | |
| chat_turn_sep = chat_sep = eos | |
| humanstr = promptA | |
| botstr = PreResponse | |
| elif prompt_type in [PromptType.wizard_mega.value, str(PromptType.wizard_mega.value), | |
| PromptType.wizard_mega.name]: | |
| preprompt = '' | |
| start = '' | |
| promptB = promptA = '%s%s' % (preprompt, start) | |
| PreInstruct = """ | |
| ### Instruction: | |
| """ | |
| PreInput = None | |
| PreResponse = """ | |
| ### Assistant: | |
| """ | |
| terminate_response = [PreResponse] | |
| chat_turn_sep = chat_sep = '\n' | |
| humanstr = PreInstruct | |
| botstr = PreResponse | |
| elif prompt_type in [PromptType.instruct_vicuna2.value, str(PromptType.instruct_vicuna2.value), | |
| PromptType.instruct_vicuna2.name]: | |
| promptA = promptB = "" if not (chat and reduced) else '' | |
| PreInstruct = """ | |
| HUMAN: | |
| """ | |
| PreInput = None | |
| PreResponse = """ | |
| ASSISTANT: | |
| """ | |
| terminate_response = [ | |
| 'HUMAN:'] # but only allow terminate after prompt is found correctly, else can't terminate | |
| chat_turn_sep = chat_sep = '\n' | |
| humanstr = PreInstruct | |
| botstr = PreResponse | |
| elif prompt_type in [PromptType.instruct_vicuna3.value, str(PromptType.instruct_vicuna3.value), | |
| PromptType.instruct_vicuna3.name]: | |
| promptA = promptB = "" if not (chat and reduced) else '' | |
| PreInstruct = """ | |
| ### User: | |
| """ | |
| PreInput = None | |
| PreResponse = """ | |
| ### Assistant: | |
| """ | |
| terminate_response = [ | |
| '### User:'] # but only allow terminate after prompt is found correctly, else can't terminate | |
| chat_turn_sep = chat_sep = '\n' | |
| humanstr = PreInstruct | |
| botstr = PreResponse | |
| elif prompt_type in [PromptType.wizard2.value, str(PromptType.wizard2.value), | |
| PromptType.wizard2.name]: | |
| # https://huggingface.co/TheBloke/WizardLM-7B-uncensored-GGML | |
| preprompt = """Below is an instruction that describes a task. Write a response that appropriately completes the request.""" if not ( | |
| chat and reduced) else '' | |
| start = '' | |
| promptB = promptA = '%s%s' % (preprompt, start) | |
| PreInstruct = """ | |
| ### Instruction: | |
| """ | |
| PreInput = None | |
| PreResponse = """ | |
| ### Response: | |
| """ | |
| terminate_response = [PreResponse] | |
| chat_turn_sep = chat_sep = '\n' | |
| humanstr = PreInstruct | |
| botstr = PreResponse | |
| elif prompt_type in [PromptType.wizard3.value, str(PromptType.wizard3.value), | |
| PromptType.wizard3.name]: | |
| # https://huggingface.co/TheBloke/wizardLM-13B-1.0-GGML | |
| preprompt = """A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.""" if not ( | |
| chat and reduced) else '' | |
| start = '' | |
| promptB = promptA = '%s%s' % (preprompt, start) | |
| PreInstruct = """USER: """ | |
| PreInput = None | |
| PreResponse = """ASSISTANT: """ | |
| terminate_response = [PreResponse] | |
| chat_turn_sep = chat_sep = '\n' | |
| humanstr = PreInstruct | |
| botstr = PreResponse | |
| elif prompt_type in [PromptType.wizard_vicuna.value, str(PromptType.wizard_vicuna.value), | |
| PromptType.wizard_vicuna.name]: | |
| preprompt = '' | |
| start = '' | |
| promptB = promptA = '%s%s' % (preprompt, start) | |
| PreInstruct = """USER: """ | |
| PreInput = None | |
| PreResponse = """ASSISTANT: """ | |
| terminate_response = [PreResponse] | |
| chat_turn_sep = chat_sep = '\n' | |
| humanstr = PreInstruct | |
| botstr = PreResponse | |
| elif prompt_type in [PromptType.instruct_simple.value, str(PromptType.instruct_simple.value), | |
| PromptType.instruct_simple.name]: | |
| promptB = promptA = '' if not (chat and reduced) else '' | |
| PreInstruct = """ | |
| ### Instruction: | |
| """ | |
| PreInput = """ | |
| ### Input: | |
| """ | |
| PreResponse = """ | |
| ### Response: | |
| """ | |
| terminate_response = None | |
| chat_turn_sep = chat_sep = '\n' | |
| humanstr = PreInstruct | |
| botstr = PreResponse | |
| elif prompt_type in [PromptType.openai.value, str(PromptType.openai.value), | |
| PromptType.openai.name]: | |
| preprompt = """The following is a conversation with an AI assistant. The assistant is helpful, creative, clever, and very friendly.""" if not ( | |
| chat and reduced) else '' | |
| start = '' | |
| promptB = promptA = '%s%s' % (preprompt, start) | |
| PreInstruct = "\nHuman: " | |
| PreInput = None | |
| PreResponse = "\nAI:" | |
| terminate_response = [PreResponse] + [" Human:", " AI:"] | |
| chat_turn_sep = chat_sep = '\n' | |
| humanstr = PreInstruct | |
| botstr = PreResponse | |
| elif prompt_type in [PromptType.gptj.value, str(PromptType.gptj.value), | |
| PromptType.gptj.name]: | |
| preprompt = "### Instruction:\n The prompt below is a question to answer, a task to complete, or a conversation to respond to; decide which and write an appropriate response." if not ( | |
| chat and reduced) else '' | |
| start = '' | |
| promptB = promptA = '%s%s' % (preprompt, start) | |
| PreInstruct = "\n### Prompt: " | |
| PreInput = None | |
| PreResponse = "\n### Response: " | |
| terminate_response = [PreResponse] + ["Prompt:", "Response:"] | |
| chat_turn_sep = chat_sep = '\n' | |
| humanstr = PreInstruct | |
| botstr = PreResponse | |
| elif prompt_type in [PromptType.openai_chat.value, str(PromptType.openai_chat.value), | |
| PromptType.openai_chat.name]: | |
| # prompting and termination all handled by endpoint | |
| preprompt = """""" | |
| start = '' | |
| promptB = promptA = '%s%s' % (preprompt, start) | |
| PreInstruct = "" | |
| PreInput = None | |
| PreResponse = "" | |
| terminate_response = [] | |
| chat_turn_sep = chat_sep = '\n' | |
| humanstr = None | |
| botstr = None | |
| elif prompt_type in [PromptType.vicuna11.value, str(PromptType.vicuna11.value), | |
| PromptType.vicuna11.name]: | |
| preprompt = """A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. """ if not ( | |
| chat and reduced) else '' | |
| start = '' | |
| promptB = promptA = '%s%s' % (preprompt, start) | |
| eos = '</s>' | |
| PreInstruct = """USER: """ | |
| PreInput = None | |
| PreResponse = """ASSISTANT:""" | |
| terminate_response = [PreResponse] | |
| chat_sep = ' ' | |
| chat_turn_sep = eos | |
| humanstr = PreInstruct | |
| botstr = PreResponse | |
| if making_context: | |
| # when making context, want it to appear as-if LLM generated, which starts with space after : | |
| PreResponse = PreResponse + ' ' | |
| else: | |
| # normally LLM adds space after this, because was how trained. | |
| # if add space here, non-unique tokenization will often make LLM produce wrong output | |
| PreResponse = PreResponse | |
| elif prompt_type in [PromptType.mptinstruct.value, str(PromptType.mptinstruct.value), | |
| PromptType.mptinstruct.name]: | |
| # https://huggingface.co/mosaicml/mpt-30b-instruct#formatting | |
| promptA = promptB = 'Below is an instruction that describes a task. Write a response that appropriately completes the request.\n' if not ( | |
| chat and reduced) else '' | |
| PreInstruct = """ | |
| ### Instruction | |
| """ | |
| PreInput = """ | |
| ### Input | |
| """ | |
| PreResponse = """ | |
| ### Response | |
| """ | |
| terminate_response = None | |
| chat_turn_sep = chat_sep = '\n' | |
| humanstr = PreInstruct | |
| botstr = PreResponse | |
| elif prompt_type in [PromptType.mptchat.value, str(PromptType.mptchat.value), | |
| PromptType.mptchat.name]: | |
| # https://huggingface.co/TheBloke/mpt-30B-chat-GGML#prompt-template | |
| promptA = promptB = """<|im_start|>system\nA conversation between a user and an LLM-based AI assistant. The assistant gives helpful and honest answers.\n<|im_end|>""" if not ( | |
| chat and reduced) else '' | |
| PreInstruct = """<|im_start|>user | |
| """ | |
| PreInput = None | |
| PreResponse = """<|im_end|><|im_start|>assistant | |
| """ | |
| terminate_response = ['<|im_end|>'] | |
| chat_sep = '' | |
| chat_turn_sep = '<|im_end|>' | |
| humanstr = PreInstruct | |
| botstr = PreResponse | |
| elif prompt_type in [PromptType.falcon.value, str(PromptType.falcon.value), | |
| PromptType.falcon.name]: | |
| promptA = promptB = "" if not (chat and reduced) else '' | |
| PreInstruct = """User: """ | |
| PreInput = None | |
| PreResponse = """Assistant:""" | |
| terminate_response = ['\nUser', "<|endoftext|>"] | |
| chat_sep = '\n\n' | |
| chat_turn_sep = '\n\n' | |
| humanstr = PreInstruct | |
| botstr = PreResponse | |
| if making_context: | |
| # when making context, want it to appear as-if LLM generated, which starts with space after : | |
| PreResponse = 'Assistant: ' | |
| else: | |
| # normally LLM adds space after this, because was how trained. | |
| # if add space here, non-unique tokenization will often make LLM produce wrong output | |
| PreResponse = PreResponse | |
| # generates_leading_space = True | |
| elif prompt_type in [PromptType.guanaco.value, str(PromptType.guanaco.value), | |
| PromptType.guanaco.name]: | |
| # https://huggingface.co/TheBloke/guanaco-65B-GPTQ | |
| promptA = promptB = "" if not (chat and reduced) else '' | |
| PreInstruct = """### Human: """ | |
| PreInput = None | |
| PreResponse = """### Assistant:""" | |
| terminate_response = ['### Human:'] # but only allow terminate after prompt is found correctly, else can't terminate | |
| chat_turn_sep = chat_sep = '\n' | |
| humanstr = PreInstruct | |
| botstr = PreResponse | |
| elif prompt_type in [PromptType.llama2.value, str(PromptType.llama2.value), | |
| PromptType.llama2.name]: | |
| PreInstruct = "" | |
| llama2_sys = "<<SYS>>\nYou are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.\n<</SYS>>\n\n" | |
| prompt = "<s>[INST] " | |
| enable_sys = False # too much safety, hurts accuracy | |
| if not (chat and reduced): | |
| if enable_sys: | |
| promptA = promptB = prompt + llama2_sys | |
| else: | |
| promptA = promptB = prompt | |
| else: | |
| promptA = promptB = '' | |
| PreInput = None | |
| PreResponse = "" | |
| terminate_response = ["[INST]", "</s>"] | |
| chat_sep = ' [/INST]' | |
| chat_turn_sep = ' </s><s>[INST] ' | |
| humanstr = PreInstruct | |
| botstr = PreResponse | |
| if making_context: | |
| PreResponse += " " | |
| else: | |
| raise RuntimeError("No such prompt_type=%s" % prompt_type) | |
| if isinstance(terminate_response, (tuple, list)): | |
| assert '' not in terminate_response, "Bad terminate_response" | |
| ret_dict = dict(promptA=promptA, promptB=promptB, PreInstruct=PreInstruct, PreInput=PreInput, | |
| PreResponse=PreResponse, terminate_response=terminate_response, chat_sep=chat_sep, | |
| chat_turn_sep=chat_turn_sep, | |
| humanstr=humanstr, botstr=botstr, | |
| generates_leading_space=generates_leading_space) | |
| if return_dict: | |
| return ret_dict, prompt_dict_error | |
| else: | |
| return tuple(list(ret_dict.values())) | |
| def generate_prompt(data_point, prompt_type, prompt_dict, chat, reduced, making_context): | |
| context = data_point.get('context') | |
| if context is None: | |
| context = '' | |
| instruction = data_point.get('instruction') | |
| input = data_point.get('input') | |
| output = data_point.get('output') | |
| prompt_type = data_point.get('prompt_type', prompt_type) | |
| prompt_dict = data_point.get('prompt_dict', prompt_dict) | |
| assert prompt_type in prompt_types, "Bad prompt type: %s" % prompt_type | |
| promptA, promptB, PreInstruct, PreInput, PreResponse, \ | |
| terminate_response, chat_sep, chat_turn_sep, humanstr, botstr, \ | |
| generates_leading_space = get_prompt(prompt_type, prompt_dict, chat, | |
| context, reduced, making_context) | |
| # could avoid if reduce=True, but too complex for parent functions to handle | |
| prompt = context | |
| if input and promptA: | |
| prompt += f"""{promptA}""" | |
| elif promptB: | |
| prompt += f"""{promptB}""" | |
| if instruction and PreInstruct is not None and input and PreInput is not None: | |
| prompt += f"""{PreInstruct}{instruction}{PreInput}{input}""" | |
| prompt = inject_chatsep(prompt_type, prompt, chat_sep=chat_sep) | |
| elif instruction and input and PreInstruct is None and PreInput is not None: | |
| prompt += f"""{PreInput}{instruction} | |
| {input}""" | |
| prompt = inject_chatsep(prompt_type, prompt, chat_sep=chat_sep) | |
| elif input and instruction and PreInput is None and PreInstruct is not None: | |
| prompt += f"""{PreInstruct}{instruction} | |
| {input}""" | |
| prompt = inject_chatsep(prompt_type, prompt, chat_sep=chat_sep) | |
| elif instruction and PreInstruct is not None: | |
| prompt += f"""{PreInstruct}{instruction}""" | |
| prompt = inject_chatsep(prompt_type, prompt, chat_sep=chat_sep) | |
| elif input and PreInput is not None: | |
| prompt += f"""{PreInput}{input}""" | |
| prompt = inject_chatsep(prompt_type, prompt, chat_sep=chat_sep) | |
| elif input and instruction and PreInput is not None: | |
| prompt += f"""{PreInput}{instruction}{input}""" | |
| prompt = inject_chatsep(prompt_type, prompt, chat_sep=chat_sep) | |
| elif input and instruction and PreInstruct is not None: | |
| prompt += f"""{PreInstruct}{instruction}{input}""" | |
| prompt = inject_chatsep(prompt_type, prompt, chat_sep=chat_sep) | |
| elif input and instruction: | |
| # i.e. for simple_instruct | |
| prompt += f"""{instruction}: {input}""" | |
| prompt = inject_chatsep(prompt_type, prompt, chat_sep=chat_sep) | |
| elif input: | |
| prompt += f"""{input}""" | |
| prompt = inject_chatsep(prompt_type, prompt, chat_sep=chat_sep) | |
| elif instruction: | |
| prompt += f"""{instruction}""" | |
| prompt = inject_chatsep(prompt_type, prompt, chat_sep=chat_sep) | |
| if PreResponse is not None: | |
| prompt += f"""{PreResponse}""" | |
| pre_response = PreResponse # Don't use strip | |
| else: | |
| pre_response = '' | |
| if output: | |
| prompt += f"""{output}""" | |
| return prompt, pre_response, terminate_response, chat_sep, chat_turn_sep | |
| def inject_chatsep(prompt_type, prompt, chat_sep=None): | |
| if chat_sep: | |
| # only add new line if structured prompt, while 'plain' is just generation of next tokens from input | |
| prompt += chat_sep | |
| return prompt | |
| class Prompter(object): | |
| def __init__(self, prompt_type, prompt_dict, debug=False, chat=False, stream_output=False, repeat_penalty=True, | |
| allowed_repeat_line_length=10): | |
| self.prompt_type = prompt_type | |
| self.prompt_dict = prompt_dict | |
| self.debug = debug | |
| self.chat = chat | |
| self.stream_output = stream_output | |
| self.repeat_penalty = repeat_penalty | |
| self.allowed_repeat_line_length = allowed_repeat_line_length | |
| self.prompt = None | |
| context = "" # not for chat context | |
| reduced = False # not for chat context | |
| making_context = False # not for chat context | |
| self.promptA, self.promptB, self.PreInstruct, self.PreInput, self.PreResponse, \ | |
| self.terminate_response, self.chat_sep, self.chat_turn_sep, self.humanstr, self.botstr, \ | |
| self.generates_leading_space = \ | |
| get_prompt(self.prompt_type, self.prompt_dict, chat, context, reduced, making_context) | |
| self.pre_response = self.PreResponse | |
| def generate_prompt(self, data_point, reduced=None): | |
| """ | |
| data_point['context'] is assumed to be like a system prompt or pre-conversation, not inserted after user prompt | |
| :param data_point: | |
| :param reduced: | |
| :return: | |
| """ | |
| reduced = data_point.get('context') not in ['', None] if reduced is None else reduced | |
| making_context = False # whether really making final prompt or just generating context | |
| prompt, _, _, _, _ = generate_prompt(data_point, self.prompt_type, self.prompt_dict, self.chat, reduced, | |
| making_context) | |
| if self.debug: | |
| print("prompt: %s" % prompt, flush=True) | |
| # if have context, should have always reduced and only preappend promptA/B here | |
| if data_point.get('context'): | |
| if data_point.get('input') and self.promptA: | |
| prompt = self.promptA + prompt | |
| elif self.promptB: | |
| prompt = self.promptB + prompt | |
| self.prompt = prompt | |
| return prompt | |
| def get_response(self, outputs, prompt=None, sanitize_bot_response=False): | |
| if isinstance(outputs, str): | |
| outputs = [outputs] | |
| if self.debug: | |
| print("output:\n%s" % '\n\n'.join(outputs), flush=True) | |
| if prompt is not None: | |
| self.prompt = prompt | |
| def clean_response(response): | |
| meaningless_words = ['<pad>', '</s>', '<|endoftext|>'] | |
| for word in meaningless_words: | |
| response = response.replace(word, "") | |
| if sanitize_bot_response: | |
| from better_profanity import profanity | |
| response = profanity.censor(response) | |
| if self.generates_leading_space and isinstance(response, str) and len(response) > 0 and response[0] == ' ': | |
| response = response[1:] | |
| return response | |
| def clean_repeats(response): | |
| lines = response.split('\n') | |
| new_lines = [] | |
| [new_lines.append(line) for line in lines if | |
| line not in new_lines or len(line) < self.allowed_repeat_line_length] | |
| if self.debug and len(lines) != len(new_lines): | |
| print("cleaned repeats: %s %s" % (len(lines), len(new_lines)), flush=True) | |
| response = '\n'.join(new_lines) | |
| return response | |
| multi_output = len(outputs) > 1 | |
| for oi, output in enumerate(outputs): | |
| if self.prompt_type in [PromptType.plain.value, str(PromptType.plain.value), PromptType.plain.name]: | |
| output = clean_response(output) | |
| elif prompt is None: | |
| # then use most basic parsing like pipeline | |
| if not self.botstr: | |
| pass | |
| elif self.botstr in output: | |
| if self.humanstr: | |
| output = clean_response(output.split(self.botstr)[1].split(self.humanstr)[0]) | |
| else: | |
| # i.e. use after bot but only up to next bot | |
| output = clean_response(output.split(self.botstr)[1].split(self.botstr)[0]) | |
| else: | |
| # output = clean_response(output) | |
| # assume just not printed yet | |
| output = "" | |
| else: | |
| # find first instance of prereponse | |
| # prompt sometimes has odd characters, that mutate length, | |
| # so can't go by length alone | |
| if self.pre_response: | |
| outputi = output.find(prompt) | |
| if outputi >= 0: | |
| output = output[outputi + len(prompt):] | |
| allow_terminate = True | |
| else: | |
| # subtraction is risky due to space offsets sometimes, so only do if necessary | |
| output = output[len(prompt) - len(self.pre_response):] | |
| # [1] to avoid repeated pre_response, just take first (after prompt - pre_response for chat) | |
| if self.pre_response in output: | |
| output = output.split(self.pre_response)[1] | |
| allow_terminate = True | |
| else: | |
| if output: | |
| print("Failure of parsing or not enough output yet: %s" % output, flush=True) | |
| allow_terminate = False | |
| else: | |
| allow_terminate = True | |
| output = output[len(prompt):] | |
| # clean after subtract prompt out, so correct removal of pre_response | |
| output = clean_response(output) | |
| if self.repeat_penalty: | |
| output = clean_repeats(output) | |
| if self.terminate_response and allow_terminate: | |
| finds = [] | |
| for term in self.terminate_response: | |
| finds.append(output.find(term)) | |
| finds = [x for x in finds if x >= 0] | |
| if len(finds) > 0: | |
| termi = finds[0] | |
| output = output[:termi] | |
| else: | |
| output = output | |
| if multi_output: | |
| # prefix with output counter | |
| output = "\n=========== Output %d\n\n" % (1 + oi) + output | |
| if oi > 0: | |
| # post fix outputs with seperator | |
| output += '\n' | |
| output = self.fix_text(self.prompt_type, output) | |
| outputs[oi] = output | |
| # join all outputs, only one extra new line between outputs | |
| output = '\n'.join(outputs) | |
| if self.debug: | |
| print("outputclean:\n%s" % '\n\n'.join(outputs), flush=True) | |
| return output | |
| def fix_text(prompt_type1, text1): | |
| if prompt_type1 == 'human_bot': | |
| # hack bug in vLLM with stopping, stops right, but doesn't return last token | |
| hfix = '<human' | |
| if text1.endswith(hfix): | |
| text1 = text1[:-len(hfix)] | |
| return text1 | |