Spaces:

minishlab
/

semantic-deduplication

Running

Pringled commited on Oct 12

Commit

2258895

•

1 Parent(s): 365d622

Updates

Files changed (1) hide show

app.py CHANGED Viewed

@@ -129,7 +129,7 @@ def perform_deduplication(
                 f"**Total documents:** {len(texts1)}\n\n"
                 f"**Duplicates found:** {num_duplicates}\n\n"
                 f"**Unique documents after deduplication:** {len(deduplicated_indices)}\n\n"
-                "-" * 50 + "\n\n"
             )
             if num_duplicates > 0:
@@ -167,6 +167,7 @@ def perform_deduplication(
                 f"**Total documents in {dataset2_name}/{dataset2_split}:** {len(texts2)}\n\n"
                 f"**Duplicates found in Dataset 2:** {num_duplicates}\n\n"
                 f"**Unique documents after deduplication:** {len(texts2) - num_duplicates}\n\n"
             )
             if num_duplicates > 0:

                 f"**Total documents:** {len(texts1)}\n\n"
                 f"**Duplicates found:** {num_duplicates}\n\n"
                 f"**Unique documents after deduplication:** {len(deduplicated_indices)}\n\n"
+                + "-" * 50 + "\n\n"
             )
             if num_duplicates > 0:
                 f"**Total documents in {dataset2_name}/{dataset2_split}:** {len(texts2)}\n\n"
                 f"**Duplicates found in Dataset 2:** {num_duplicates}\n\n"
                 f"**Unique documents after deduplication:** {len(texts2) - num_duplicates}\n\n"
+                + "-" * 50 + "\n\n"
             )
             if num_duplicates > 0: