Spaces:
Runtime error
Runtime error
| import gradio as gr | |
| from bs4 import BeautifulSoup | |
| import json | |
| import time | |
| import os | |
| from transformers import AutoTokenizer, pipeline | |
| example1 = '''<!DOCTYPE html> | |
| <html> | |
| <head> | |
| <meta charset="UTF-8"> | |
| <title>Contact Form</title> | |
| </head> | |
| <body> | |
| <h1>Contact Form</h1> | |
| <form action="/submit-form" method="POST"> | |
| <label for="name">Name:</label> | |
| <input type="text" id="name" name="name" required> | |
| <br> | |
| <label for="email">Email:</label> | |
| <input type="email" id="email" name="email" required> | |
| <br> | |
| <label for="location">Location:</label> | |
| <input type="text" id="location" name="location" required> | |
| <br> | |
| <label for="github">GitHub:</label> | |
| <input type="url" id="github" name="github" required> | |
| <br> | |
| <label for="linkedin">LinkedIn:</label> | |
| <input type="url" id="linkedin" name="linkedin" required> | |
| <br> | |
| <label for="phone">Phone:</label> | |
| <input type="tel" id="phone" name="phone" required> | |
| <br><br> | |
| <input type="submit" value="Submit"> | |
| </form> | |
| </body> | |
| </html> | |
| ''' | |
| solution1 = '''{ | |
| "name": "Ana Guida", | |
| "email": "[email protected]", | |
| "location": "Amsterdam, Netherlands", | |
| "github": "https://github.com/34kmddfn", | |
| "linkedin": "https://www.linkedin.com/in/ana-rguida/", | |
| "phone": "+351 928 169 341" | |
| }''' | |
| example2 = '''<!DOCTYPE html> | |
| <html> | |
| <head> | |
| <title>Resume Form</title> | |
| </head> | |
| <body> | |
| <form action="/" method="POST"> | |
| <label>What kind of pet do you have?</label> | |
| <br> | |
| <input type="radio" id="dog" name="pet" value="dog"> | |
| <label for="dog">Dog</label> | |
| <br> | |
| <input type="radio" id="cat" name="pet" value="cat"> | |
| <label for="cat">Cat</label> | |
| <br> | |
| <input type="radio" id="other" name="pet" value="other"> | |
| <label for="other">Other</label> | |
| <br><br> | |
| <label>What color is your pet?</label> | |
| <br> | |
| <input type="checkbox" id="black" name="color" value="black"> | |
| <label for="black">Black</label> | |
| <br> | |
| <input type="checkbox" id="white" name="color" value="white"> | |
| <label for="white">White</label> | |
| <br> | |
| <input type="checkbox" id="brown" name="color" value="brown"> | |
| <label for="brown">Brown</label> | |
| <br><br> | |
| <input type="submit" value="Submit"> | |
| </form> | |
| </body> | |
| </html> | |
| ''' | |
| solution2 = '''{ | |
| "pet": "dog", | |
| "color": [ | |
| "black", | |
| "brown" | |
| ] | |
| }''' | |
| example3 = '''<!DOCTYPE html> | |
| <html> | |
| <head> | |
| <title>Create account Form</title> | |
| </head> | |
| <body> | |
| <form action="/" method="POST"> | |
| <label for="name">Name:</label> | |
| <input type="text" id="name" name="name" required> | |
| <br> | |
| <label for="country">Select your country:</label> | |
| <br> | |
| <select id="country" name="country"> | |
| <option value="usa">USA</option> | |
| <option value="uk">UK</option> | |
| <option value="germany">Germany</option> | |
| <option value="japan">Japan</option> | |
| </select> | |
| <br><br> | |
| <label for="birthday">Select your birthday:</label> | |
| <br> | |
| <input type="date" id="birthday" name="birthday"> | |
| <br><br> | |
| <input type="submit" value="Submit"> | |
| </form> | |
| </body> | |
| </html> | |
| ''' | |
| solution3 = '''{ | |
| "name": "Mike", | |
| "country": "Germany", | |
| "birthday": "1990-05-07" | |
| }''' | |
| models = { | |
| "model_n1": "sileod/deberta-v3-base-tasksource-nli", | |
| # "model_n2": "roberta-large-mnli", | |
| # "model_n3": "facebook/bart-large-mnli", | |
| # "model_n4": "cross-encoder/nli-deberta-v3-xsmall" | |
| } | |
| def find_form_fields_from_file(file): | |
| with open(file.name, "r") as f: | |
| content = f.read() | |
| return find_form_fields(content) | |
| def find_form_fields(html_content): | |
| soup = BeautifulSoup(html_content, 'html.parser') | |
| # find all form tags | |
| forms = soup.find_all('form') | |
| form_fields = [] | |
| for form in forms: | |
| # find all input and select tags within each form | |
| input_tags = form.find_all('input') | |
| select_tags = form.find_all('select') | |
| for tag in input_tags: | |
| form_fields.append(str(tag)) | |
| for tag in select_tags: | |
| form_fields.append(str(tag)) | |
| # Convert the list to a single string for display | |
| return form_fields | |
| def load_json(json_file): | |
| with open(json_file, 'r') as f: | |
| data = json.load(f) | |
| return data | |
| def classify_lines(text, candidate_labels, model_name): | |
| start_time = time.time() # Start measuring time | |
| classifier = pipeline('zero-shot-classification', model=model_name) | |
| # Check if the text is already a list or if it needs splitting | |
| if isinstance(text, list): | |
| lines = text | |
| else: | |
| lines = text.split('\n') | |
| classified_lines = [] | |
| for line in lines: | |
| if line.strip() and (line.strip().startswith("<input") or line.strip().startswith("<select") )and 'hidden' not in line.lower(): | |
| # Skip empty lines, classify lines starting with "<input", and exclude lines with 'hidden' | |
| results = classifier(line, candidate_labels=candidate_labels) | |
| top_classifications = results['labels'][:2] # Get the top two classifications | |
| top_scores = results['scores'][:2] # Get the top two scores | |
| classified_lines.append((line, list(zip(top_classifications, top_scores)))) | |
| end_time = time.time() # Stop measuring time | |
| execution_time = end_time - start_time # Calculate execution time | |
| return classified_lines, execution_time | |
| def classify_lines_json(text, json_content, candidate_labels, model_name): | |
| start_time = time.time() # Start measuring time | |
| classifier = pipeline('zero-shot-classification', model=model_name) | |
| # Check if the text is already a list or if it needs splitting | |
| if isinstance(text, list): | |
| lines = text | |
| else: | |
| lines = text.split('\n') | |
| # Open the output.html file in write mode | |
| output_content = [] | |
| last_input = "None" | |
| max_index = -1 | |
| for i, line in enumerate(lines): | |
| if line.strip() and (line.strip().startswith("<input") or line.strip().startswith("<select") or line.strip().startswith("<option") ) and 'hidden' not in line.lower(): | |
| # Skip empty lines, classify lines starting with "<input", and exclude lines with 'hidden' | |
| results = classifier(line, candidate_labels=["text", "radio", "checkbox", "button", "date", "select"]) | |
| if results['labels'][0] == "text" or results['labels'][0] == "date": | |
| # print("text") | |
| last_input = "text/date" | |
| input_results = classifier(line, candidate_labels=candidate_labels) | |
| top_classifications = input_results['labels'][:2] # Get the top two classifications | |
| top_scores = input_results['scores'][:2] # Get the top two scores | |
| line = line + f"<!-- Input: <{json_content[top_classifications[0]]}> - certainty: {format(top_scores[0], '.2f')} -->" | |
| elif results['labels'][0] == "button": | |
| # print("button") | |
| last_input = "button" | |
| line = line + f"<!-- Input: <{results['labels'][0]}> - certainty: {format(results['scores'][0], '.2f')} -->" | |
| elif results['labels'][0] == "radio": | |
| # print("radio") | |
| if(last_input == "radio"): | |
| radio_options.append(line) | |
| radio_options_i.append(i) | |
| else: | |
| radio_options = [line] | |
| radio_options_i = [i] | |
| radio_results_list = [] | |
| last_input = "radio" | |
| input_results = classifier(line, candidate_labels=candidate_labels) | |
| top_classifications = input_results['labels'][:2] # Get the top two classifications | |
| top_scores = input_results['scores'][:2] # Get the top two scores | |
| radio_results = classifier(line, candidate_labels=[json_content[top_classifications[0]]]) | |
| radio_results_list.append(radio_results) | |
| # Get the scores from the radio_results_list | |
| scores = [result['scores'][0] for result in radio_results_list] | |
| previous_max_index = max_index | |
| # Find the index of the maximum score | |
| max_index = scores.index(max(scores)) | |
| if previous_max_index != max_index: | |
| line_selected = radio_options[previous_max_index] | |
| real_index = radio_options_i[previous_max_index] | |
| if real_index < len(output_content): | |
| output_content[real_index] = line_selected | |
| line_selected = radio_options[max_index] | |
| line_selected = line_selected + f"<!-- Input: <{results['labels'][0]}> - certainty: {format(results['scores'][0], '.2f')}. LINE TO SELECT: <{radio_results['labels'][0]}> - certainty: {format(max(scores), '.2f')} -->" | |
| real_index = radio_options_i[max_index] | |
| if real_index < len(output_content): | |
| output_content[real_index] = line_selected | |
| else: | |
| line = line_selected | |
| elif results['labels'][0] == "checkbox": | |
| # print("checkbox") | |
| last_input = "checkbox" | |
| input_results = classifier(line, candidate_labels=candidate_labels) | |
| top_classifications = input_results['labels'][:2] # Get the top two classifications | |
| top_scores = input_results['scores'][:2] # Get the top two scores | |
| checkbox_results = classifier(line, candidate_labels=[json_content[top_classifications[0]]]) | |
| if checkbox_results['scores'][0] > 0.8: | |
| line = line + f"<!-- Input: <{results['labels'][0]}> - certainty: {format(results['scores'][0], '.2f')}. LINE TO SELECT: <{checkbox_results['labels'][0]}> - certainty: {format(checkbox_results['scores'][0], '.2f')} -->" | |
| else: #elif results['labels'][0] == "select" or results['labels'][0] == "option": | |
| # print("select") | |
| if(last_input == "select"): | |
| select_options.append(line) | |
| select_options_i.append(i) | |
| else: | |
| select_options = [line] | |
| select_options_i = [i] | |
| select_results_list = [] | |
| last_input = "select" | |
| input_results = classifier(line, candidate_labels=candidate_labels) | |
| top_classifications = input_results['labels'][:2] # Get the top two classifications | |
| top_scores = input_results['scores'][:2] # Get the top two scores | |
| select_results = classifier(line, candidate_labels=[json_content[top_classifications[0]]]) | |
| select_results_list.append(select_results) | |
| # Get the scores from the select_results_list | |
| scores = [result['scores'][0] for result in select_results_list] | |
| previous_max_index = max_index | |
| # Find the index of the maximum score | |
| max_index = scores.index(max(scores)) | |
| if previous_max_index != max_index: | |
| line_selected = select_options[previous_max_index] | |
| real_index = select_options_i[previous_max_index] | |
| if real_index < len(output_content): | |
| output_content[real_index] = line_selected | |
| line_selected = select_options[max_index] | |
| line_selected = line_selected + f"<!-- Input: <{results['labels'][0]}> - certainty: {format(results['scores'][0], '.2f')}. LINE TO SELECT: <{select_results['labels'][0]}> - certainty: {format(max(scores), '.2f')} -->" | |
| real_index = select_options_i[max_index] | |
| if real_index < len(output_content): | |
| output_content[real_index] = line_selected | |
| else: | |
| line = line_selected | |
| output_content.append(line) | |
| end_time = time.time() # Stop measuring time | |
| execution_time = end_time - start_time # Calculate execution time | |
| return output_content, execution_time | |
| def retrieve_fields(data, path=''): | |
| """Recursively retrieve all fields from a given JSON structure and prompt for filling.""" | |
| fields = {} | |
| # If the data is a dictionary | |
| if isinstance(data, dict): | |
| for key, value in data.items(): | |
| # Construct the updated path for nested structures | |
| new_path = f"{path}.{key}" if path else key | |
| fields.update(retrieve_fields(value, new_path)) | |
| # If the data is a list, iterate over its items | |
| elif isinstance(data, list): | |
| for index, item in enumerate(data): | |
| new_path = f"{path}[{index}]" | |
| fields.update(retrieve_fields(item, new_path)) | |
| # If the data is a simple type (str, int, etc.) | |
| else: | |
| prompt = f"Please fill in the {path} field." if not data else data | |
| fields[path] = prompt | |
| return fields | |
| def retrieve_fields_from_file(file_path): | |
| """Load JSON data from a file, then retrieve all fields and prompt for filling.""" | |
| with open(file_path.name, 'r') as f: | |
| data = f.read() | |
| return retrieve_fields(json.loads(data)) | |
| def process_files(html_file, json_file): | |
| #html_content = open_html(html_file) | |
| #print(html_file) | |
| html_inputs = find_form_fields(html_file) | |
| #print(json_file) | |
| json_content = retrieve_fields(json.loads(json_file)) | |
| #Classificar os inputs do json para ver em que tipo de input ["text", "radio", "checkbox", "button", "date"] | |
| # Classify lines and measure execution time | |
| for model_name in models.values(): | |
| tokenizer = AutoTokenizer.from_pretrained(model_name) | |
| #html_classified_lines, html_execution_time = classify_lines(html_inputs, ["text", "radio", "checkbox", "button", "date", "select"], model_name) | |
| json_classified_lines, json_execution_time = classify_lines_json(html_file, json_content, list(json_content.keys()), model_name) | |
| # print(str(html_execution_time) + " - " + str(html_classified_lines)) | |
| # print(str(json_execution_time) + " - " + str(json_classified_lines)) | |
| #print(type(json_classified_lines)) | |
| #return '\n'.join(map(str, html_classified_lines)) | |
| return '\n'.join(map(str, json_classified_lines)) | |
| HF_TOKEN = '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' | |
| hf_writer = gr.HuggingFaceDatasetSaver(HF_TOKEN, "form_matcher_demo_flagged") | |
| iface = gr.Interface(fn=process_files, | |
| inputs=[gr.Textbox(lines = 20, max_lines = 1000, label="Upload HTML File"), gr.Textbox(lines = 20, max_lines = 1000, label="Upload JSON File")], | |
| outputs=gr.Textbox(lines = 20, max_lines = 1000, label="Output"), | |
| examples=[ | |
| [example1, solution1], | |
| [example2, solution2], | |
| [example3, solution3], | |
| ], | |
| allow_flagging="manual", | |
| flagging_callback=hf_writer | |
| ) | |
| iface.launch() | |