Spaces:
Sleeping
Sleeping
lukestanley
commited on
Commit
•
ac30aa7
1
Parent(s):
759e510
Refactor code: Import libraries, compare moderation APIs, and remove unused code
Browse files
learn.py
CHANGED
@@ -1,7 +1,18 @@
|
|
1 |
# %%
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2 |
import pandas as pd
|
3 |
from datasets import load_dataset
|
4 |
from detoxify import Detoxify
|
|
|
|
|
|
|
|
|
5 |
predict_model = Detoxify('original-small')
|
6 |
dataset = load_dataset("tasksource/jigsaw")
|
7 |
|
@@ -20,13 +31,6 @@ predict_model.predict("You suck, that is not Markdown!") # Also accepts an array
|
|
20 |
'insult': 0.7787105,
|
21 |
'identity_attack': 0.007936229}
|
22 |
|
23 |
-
# %%
|
24 |
-
import asyncio
|
25 |
-
import json
|
26 |
-
import time
|
27 |
-
import os
|
28 |
-
import hashlib
|
29 |
-
from functools import wraps
|
30 |
|
31 |
|
32 |
_in_memory_cache = {}
|
@@ -127,7 +131,6 @@ def timeit(func):
|
|
127 |
|
128 |
|
129 |
|
130 |
-
# %%
|
131 |
|
132 |
@cache("toxicity")
|
133 |
def cached_toxicity_prediction(comments):
|
|
|
1 |
# %%
|
2 |
+
import asyncio
|
3 |
+
import json
|
4 |
+
import time
|
5 |
+
import os
|
6 |
+
import hashlib
|
7 |
+
from functools import wraps
|
8 |
+
|
9 |
import pandas as pd
|
10 |
from datasets import load_dataset
|
11 |
from detoxify import Detoxify
|
12 |
+
|
13 |
+
# TODO: Compare OpenAI's moderation API to Detoxify
|
14 |
+
|
15 |
+
|
16 |
predict_model = Detoxify('original-small')
|
17 |
dataset = load_dataset("tasksource/jigsaw")
|
18 |
|
|
|
31 |
'insult': 0.7787105,
|
32 |
'identity_attack': 0.007936229}
|
33 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
34 |
|
35 |
|
36 |
_in_memory_cache = {}
|
|
|
131 |
|
132 |
|
133 |
|
|
|
134 |
|
135 |
@cache("toxicity")
|
136 |
def cached_toxicity_prediction(comments):
|