add cc file

CC-MAIN-20231128083443-20231128113443-00000.warc.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a754b9a848ffaf00ba1621ad4577a31ce3176b058c3ed21cb3665954b7ae7cf
+size 1208969113

app.py

@@ -0,0 +1,24 @@
+from itertools import islice
+
+import gradio as gr
+from datatrove.executor.local import LocalPipelineExecutor
+from datatrove.pipeline.base import PipelineStep
+from datatrove.pipeline.extractors import Trafilatura
+from datatrove.pipeline.filters import (
+    C4QualityFilter,
+    FineWebQualityFilter,
+    GopherQualityFilter,
+    GopherRepetitionFilter,
+    LanguageFilter,
+    URLFilter,
+)
+from datatrove.pipeline.readers import WarcReader
+from datatrove.pipeline.writers.jsonl import JsonlWriter
+
+
+def run(input):
+    return "wip"
+
+
+demo = gr.Interface(run, inputs=[gr.Textbox()], outputs=[gr.Textbox()])
+demo.launch()

requirements.txt

@@ -0,0 +1 @@
+datatrove[all]