lukestanley commited on
Commit
ac30aa7
1 Parent(s): 759e510

Refactor code: Import libraries, compare moderation APIs, and remove unused code

Browse files
Files changed (1) hide show
  1. learn.py +11 -8
learn.py CHANGED
@@ -1,7 +1,18 @@
1
  # %%
 
 
 
 
 
 
 
2
  import pandas as pd
3
  from datasets import load_dataset
4
  from detoxify import Detoxify
 
 
 
 
5
  predict_model = Detoxify('original-small')
6
  dataset = load_dataset("tasksource/jigsaw")
7
 
@@ -20,13 +31,6 @@ predict_model.predict("You suck, that is not Markdown!") # Also accepts an array
20
  'insult': 0.7787105,
21
  'identity_attack': 0.007936229}
22
 
23
- # %%
24
- import asyncio
25
- import json
26
- import time
27
- import os
28
- import hashlib
29
- from functools import wraps
30
 
31
 
32
  _in_memory_cache = {}
@@ -127,7 +131,6 @@ def timeit(func):
127
 
128
 
129
 
130
- # %%
131
 
132
  @cache("toxicity")
133
  def cached_toxicity_prediction(comments):
 
1
  # %%
2
+ import asyncio
3
+ import json
4
+ import time
5
+ import os
6
+ import hashlib
7
+ from functools import wraps
8
+
9
  import pandas as pd
10
  from datasets import load_dataset
11
  from detoxify import Detoxify
12
+
13
+ # TODO: Compare OpenAI's moderation API to Detoxify
14
+
15
+
16
  predict_model = Detoxify('original-small')
17
  dataset = load_dataset("tasksource/jigsaw")
18
 
 
31
  'insult': 0.7787105,
32
  'identity_attack': 0.007936229}
33
 
 
 
 
 
 
 
 
34
 
35
 
36
  _in_memory_cache = {}
 
131
 
132
 
133
 
 
134
 
135
  @cache("toxicity")
136
  def cached_toxicity_prediction(comments):