Spaces:

sergeipetrov
/

webhook-space

Runtime error

App Files Files Community

plaggy commited on Mar 1

Commit

187981b

•

1 Parent(s): ce236eb

html template

Browse files

Files changed (11) hide show

Dockerfile +1 -1
__pycache__/main.cpython-310.pyc +0 -0
__pycache__/models.cpython-310.pyc +0 -0
chunk_config.json → configs/chunk_config.json +0 -0
embed_config.json → configs/embed_config.json +0 -0
src/main.py → main.py +17 -12
src/models.py → models.py +2 -2
requirements.txt +2 -1
src/__init__.py +0 -0
style.css → static/style.css +0 -0
home.html → templates/index.html +4 -4

Dockerfile CHANGED Viewed

@@ -13,4 +13,4 @@ RUN pip install --no-cache-dir --upgrade -r requirements.txt
 COPY --chown=user . .
-CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "7860"]


13
14	COPY --chown=user . .
15
16	+ CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

__pycache__/main.cpython-310.pyc ADDED Viewed

Binary file (6.21 kB). View file

__pycache__/models.cpython-310.pyc ADDED Viewed

Binary file (1.79 kB). View file

chunk_config.json → configs/chunk_config.json RENAMED Viewed

File without changes

embed_config.json → configs/embed_config.json RENAMED Viewed

File without changes

src/main.py → main.py RENAMED Viewed

@@ -7,8 +7,10 @@ import os
 import tempfile
 import requests
-from fastapi import FastAPI, BackgroundTasks
-from fastapi.responses import FileResponse
 from aiohttp import ClientSession
 from langchain.text_splitter import RecursiveCharacterTextSplitter
@@ -16,7 +18,7 @@ from datasets import Dataset, load_dataset
 from tqdm import tqdm
 from tqdm.asyncio import tqdm_asyncio
-from src.models import chunk_config, embed_config, WebhookPayload
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -35,15 +37,18 @@ INPUT_SPLITS = os.getenv("INPUT_SPLITS")
 # name of column to load from input dataset
 INPUT_TEXT_COL = os.getenv("INPUT_TEXT_COL")
-INPUT_SPLITS = [spl.strip() for spl in INPUT_SPLITS.split(",") if spl]
 app = FastAPI()
 app.state.seen_Sha = set()
-@app.get("/")
-async def home():
-    return FileResponse("home.html")
 @app.post("/webhook")
@@ -56,13 +61,13 @@ async def post_webhook(
         and payload.event.scope.startswith("repo.content")
         and payload.repo.type == "dataset"
         # webhook posts multiple requests with the same update, this addresses that
-        and payload.repo.headSha not in app.state.last_Sha
     ):
         # no-op
         logger.info("Update detected, no action taken")
         return {"processed": False}
-    app.state.last_Sha.add(payload.repo.headSha)
     task_queue.add_task(chunk_dataset, ds_name=payload.repo.name)
     task_queue.add_task(embed_dataset, ds_name=CHUNKED_DS_NAME)
@@ -206,6 +211,6 @@ def embed_dataset(ds_name):
 # For debugging
-# import uvicorn
-# if __name__ == "__main__":
-#     uvicorn.run(app, host="0.0.0.0", port=7860)

 import tempfile
 import requests
+from fastapi import FastAPI, Request, BackgroundTasks
+from fastapi.responses import HTMLResponse
+from fastapi.staticfiles import StaticFiles
+from fastapi.templating import Jinja2Templates
 from aiohttp import ClientSession
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from tqdm import tqdm
 from tqdm.asyncio import tqdm_asyncio
+from models import chunk_config, embed_config, WebhookPayload
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 # name of column to load from input dataset
 INPUT_TEXT_COL = os.getenv("INPUT_TEXT_COL")
+# INPUT_SPLITS = [spl.strip() for spl in INPUT_SPLITS.split(",") if spl]
 app = FastAPI()
 app.state.seen_Sha = set()
+app.mount("/static", StaticFiles(directory="static"), name="static")
+templates = Jinja2Templates(directory="templates")
+@app.get("/", response_class=HTMLResponse)
+async def root(request: Request):
+    return templates.TemplateResponse(request=request, name="index.html")
+    # return FileResponse("/Users/spetrov/Documents/PROJECTS/hub_etl_pipeline/auto-chunk-embed/templates/index.html")
 @app.post("/webhook")
         and payload.event.scope.startswith("repo.content")
         and payload.repo.type == "dataset"
         # webhook posts multiple requests with the same update, this addresses that
+        and payload.repo.headSha not in app.state.seen_Sha
     ):
         # no-op
         logger.info("Update detected, no action taken")
         return {"processed": False}
+    app.state.seen_Sha.add(payload.repo.headSha)
     task_queue.add_task(chunk_dataset, ds_name=payload.repo.name)
     task_queue.add_task(embed_dataset, ds_name=CHUNKED_DS_NAME)
 # For debugging
+import uvicorn
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=7860)

src/models.py → models.py RENAMED Viewed

@@ -34,10 +34,10 @@ class WebhookPayload(BaseModel):
 	repo: WebhookPayloadRepo
-with open(os.path.join(os.getcwd(), "chunk_config.json")) as c:
 	data = json.load(c)
 	chunk_config = ChunkConfig.model_validate_json(json.dumps(data))
-with open(os.path.join(os.getcwd(), "embed_config.json")) as c:
 	data = json.load(c)
 	embed_config = EmbedConfig.model_validate_json(json.dumps(data))

 	repo: WebhookPayloadRepo
+with open(os.path.join(os.getcwd(), "configs/chunk_config.json")) as c:
 	data = json.load(c)
 	chunk_config = ChunkConfig.model_validate_json(json.dumps(data))
+with open(os.path.join(os.getcwd(), "configs/embed_config.json")) as c:
 	data = json.load(c)
 	embed_config = EmbedConfig.model_validate_json(json.dumps(data))

requirements.txt CHANGED Viewed

@@ -1,4 +1,4 @@
-fastapi==0.104.*
 requests==2.31.*
 huggingface-hub==0.20.*
 uvicorn[standard]==0.17.*
@@ -7,3 +7,4 @@ datasets==2.16.*
 langchain==0.0.*
 aiohttp==3.8.*
 spacy==3.7.*

+fastapi==0.110.*
 requests==2.31.*
 huggingface-hub==0.20.*
 uvicorn[standard]==0.17.*
 langchain==0.0.*
 aiohttp==3.8.*
 spacy==3.7.*
+jinja2==3.1.*

src/__init__.py DELETED Viewed

File without changes

style.css → static/style.css RENAMED Viewed

File without changes

home.html → templates/index.html RENAMED Viewed

@@ -1,10 +1,10 @@
 <!DOCTYPE html>
 <html>
 	<head>
-		<meta charset="utf-8" />
-		<meta name="viewport" content="width=device-width" />
 		<title>Auto chunking and embedding</title>
-		<link rel="stylesheet" href="style.css" />
 	</head>
     <body>
         <div class="card">
@@ -12,7 +12,7 @@
             <p>This is a webhook space to chunk and embed a dataset when it changes.</p>
-            <p>Use this as a reference <a href="https://huggingface.co/docs/hub/webhooks-guide-auto-retrain" target="_blank">here</a>!</p>
         </div>
     </body>
 </html>

 <!DOCTYPE html>
 <html>
 	<head>
+		<meta charset="utf-8">
+		<meta name="viewport" content="width=device-width">
 		<title>Auto chunking and embedding</title>
+		<link href="{{ url_for('static', path='/style.css') }}" rel="stylesheet">
 	</head>
     <body>
         <div class="card">
             <p>This is a webhook space to chunk and embed a dataset when it changes.</p>
+            <p>Use <a href="https://huggingface.co/docs/hub/webhooks-guide-auto-retrain" target="_blank">this</a> as a reference</p>
         </div>
     </body>
 </html>