Wauplin HF staff commited on
Commit
ad2e859
1 Parent(s): 51aeff1
Files changed (6) hide show
  1. .gitignore +139 -0
  2. Dockerfile +16 -0
  3. README.md +2 -0
  4. app.py +194 -0
  5. home.html +18 -0
  6. requirements.txt +4 -0
.gitignore ADDED
@@ -0,0 +1,139 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib64/
18
+ parts/
19
+ sdist/
20
+ var/
21
+ wheels/
22
+ pip-wheel-metadata/
23
+ share/python-wheels/
24
+ *.egg-info/
25
+ .installed.cfg
26
+ *.egg
27
+ MANIFEST
28
+
29
+ # PyInstaller
30
+ # Usually these files are written by a python script from a template
31
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
32
+ *.manifest
33
+ *.spec
34
+
35
+ # Installer logs
36
+ pip-log.txt
37
+ pip-delete-this-directory.txt
38
+
39
+ # Unit test / coverage reports
40
+ htmlcov/
41
+ .tox/
42
+ .nox/
43
+ .coverage
44
+ .coverage.*
45
+ .cache
46
+ nosetests.xml
47
+ coverage.xml
48
+ *.cover
49
+ *.py,cover
50
+ .hypothesis/
51
+ .pytest_cache/
52
+
53
+ # Translations
54
+ *.mo
55
+ *.pot
56
+
57
+ # Django stuff:
58
+ *.log
59
+ local_settings.py
60
+ db.sqlite3
61
+ db.sqlite3-journal
62
+
63
+ # Flask stuff:
64
+ instance/
65
+ .webassets-cache
66
+
67
+ # Scrapy stuff:
68
+ .scrapy
69
+
70
+ # Sphinx documentation
71
+ docs/_build/
72
+
73
+ # PyBuilder
74
+ target/
75
+
76
+ # Jupyter Notebook
77
+ .ipynb_checkpoints
78
+
79
+ # IPython
80
+ profile_default/
81
+ ipython_config.py
82
+
83
+ # pyenv
84
+ .python-version
85
+
86
+ # pipenv
87
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
88
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
89
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
90
+ # install all needed dependencies.
91
+ #Pipfile.lock
92
+
93
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow
94
+ __pypackages__/
95
+
96
+ # Celery stuff
97
+ celerybeat-schedule
98
+ celerybeat.pid
99
+
100
+ # SageMath parsed files
101
+ *.sage.py
102
+
103
+ # Environments
104
+ .env
105
+ .venv
106
+ .venv*
107
+ env/
108
+ venv/
109
+ ENV/
110
+ env.bak/
111
+ venv.bak/
112
+ .venv*
113
+
114
+ # Spyder project settings
115
+ .spyderproject
116
+ .spyproject
117
+
118
+ # Rope project settings
119
+ .ropeproject
120
+
121
+ # mkdocs documentation
122
+ /site
123
+
124
+ # mypy
125
+ .mypy_cache/
126
+ .dmypy.json
127
+ dmypy.json
128
+
129
+ # Pyre type checker
130
+ .pyre/
131
+ .vscode/
132
+ .idea/
133
+
134
+ .DS_Store
135
+
136
+ .venv
137
+ .vscode
138
+ __pycache__
139
+
Dockerfile ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM python:3.10
2
+
3
+ RUN useradd -m -u 1000 user
4
+ USER user
5
+ ENV HOME=/home/user \
6
+ PATH=/home/user/.local/bin:$PATH
7
+
8
+ WORKDIR $HOME/app
9
+
10
+ COPY --chown=user requirements.txt requirements.txt
11
+ RUN pip install --no-cache-dir --upgrade -r requirements.txt
12
+
13
+ COPY --chown=user . .
14
+
15
+ CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
16
+
README.md CHANGED
@@ -8,3 +8,5 @@ pinned: false
8
  ---
9
 
10
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
8
  ---
9
 
10
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
11
+
12
+ Project structure taken from https://huggingface.co/spaces/huggingface-projects/auto-retrain.
app.py ADDED
@@ -0,0 +1,194 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from pathlib import Path
3
+ from typing import Literal, Optional
4
+
5
+ from fastapi import BackgroundTasks, FastAPI, Header, HTTPException
6
+ from fastapi.responses import FileResponse
7
+ from huggingface_hub import (
8
+ CommitOperationAdd,
9
+ CommitOperationDelete,
10
+ comment_discussion,
11
+ create_commit,
12
+ create_repo,
13
+ delete_repo,
14
+ snapshot_download,
15
+ )
16
+ from pydantic import BaseModel
17
+ from requests import HTTPError
18
+
19
+ WEBHOOK_SECRET = os.getenv("WEBHOOK_SECRET")
20
+ HF_TOKEN = os.getenv("HF_TOKEN")
21
+
22
+
23
+ class WebhookPayloadEvent(BaseModel):
24
+ action: Literal["create", "update", "delete"]
25
+ scope: str
26
+
27
+
28
+ class WebhookPayloadRepo(BaseModel):
29
+ type: Literal["dataset", "model", "space"]
30
+ name: str
31
+ private: bool
32
+
33
+
34
+ class WebhookPayloadDiscussion(BaseModel):
35
+ num: int
36
+ isPullRequest: bool
37
+
38
+
39
+ class WebhookPayload(BaseModel):
40
+ event: WebhookPayloadEvent
41
+ repo: WebhookPayloadRepo
42
+ discussion: Optional[WebhookPayloadDiscussion]
43
+
44
+
45
+ app = FastAPI()
46
+
47
+
48
+ @app.get("/")
49
+ async def home():
50
+ return FileResponse("home.html")
51
+
52
+
53
+ @app.post("/webhook")
54
+ async def post_webhook(
55
+ payload: WebhookPayload,
56
+ task_queue: BackgroundTasks,
57
+ x_webhook_secret: Optional[str] = Header(default=None),
58
+ ):
59
+ # Taken from https://huggingface.co/spaces/huggingface-projects/auto-retrain
60
+ if x_webhook_secret is None:
61
+ raise HTTPException(401)
62
+ if x_webhook_secret != WEBHOOK_SECRET:
63
+ raise HTTPException(403)
64
+
65
+ if payload.repo.type != "space":
66
+ raise HTTPException(400, f"Must be a Space, not {payload.repo.type}")
67
+
68
+ if not payload.event.scope.startswith("discussion"):
69
+ return "Not a discussion"
70
+
71
+ if payload.discussion is None:
72
+ return "Couldn't parse 'payload.discussion'"
73
+
74
+ if not payload.discussion.isPullRequest:
75
+ return "Not a Pull Request"
76
+
77
+ if payload.event.action == "create" or payload.event.action == "update":
78
+ task_queue.add_task(
79
+ sync_ci_space,
80
+ space_id=payload.repo.name,
81
+ pr_num=payload.discussion.num,
82
+ private=payload.repo.private,
83
+ )
84
+ elif payload.event.action == "delete":
85
+ task_queue.add_task(
86
+ delete_ci_space,
87
+ space_id=payload.repo.name,
88
+ pr_num=payload.discussion.num,
89
+ )
90
+ else:
91
+ return f"Couldn't handle action {payload.event.action}"
92
+
93
+ return "Processed"
94
+
95
+
96
+ def sync_ci_space(space_id: str, pr_num: int, private=bool) -> None:
97
+ # Create a temporary space for CI if didn't exist
98
+ ci_space_id = _get_ci_space_id(space_id=space_id, pr_num=pr_num)
99
+
100
+ try:
101
+ create_repo(ci_space_id, repo_type="space", private=private, token=HF_TOKEN)
102
+ is_new = True
103
+ except HTTPError as err:
104
+ if err.response.status_code == 409: # already exists
105
+ is_new = False
106
+ else:
107
+ raise
108
+
109
+ # Download space codebase from PR revision
110
+ snapshot_path = snapshot_download(
111
+ repo_id=space_id,
112
+ revision=f"refs/pr/{pr_num}",
113
+ repo_type="space",
114
+ token=HF_TOKEN,
115
+ )
116
+
117
+ # Sync space codebase with PR revision
118
+ operations = [CommitOperationDelete("/")] # little aggressive but works
119
+ operations += [
120
+ CommitOperationAdd(
121
+ path_in_repo=str(filepath.relative_to(snapshot_path)),
122
+ path_or_fileobj=filepath,
123
+ )
124
+ for filepath in Path(snapshot_path).glob("*/**")
125
+ if filepath.is_file()
126
+ ]
127
+ create_commit(
128
+ repo_id=ci_space_id,
129
+ repo_type="space",
130
+ operations=operations,
131
+ commit_message=f"Sync CI Space with PR {pr_num}.",
132
+ token=HF_TOKEN,
133
+ )
134
+
135
+ # Post a comment on the PR
136
+ notify_pr(space_id=space_id, pr_num=pr_num, action="create" if is_new else "update")
137
+
138
+
139
+ def delete_ci_space(space_id: str, pr_num: int) -> None:
140
+ # Delete
141
+ ci_space_id = _get_ci_space_id(space_id=space_id, pr_num=pr_num)
142
+ delete_repo(repo_id=ci_space_id, repo_type="space", token=HF_TOKEN)
143
+
144
+ # Notify about deletion
145
+ notify_pr(space_id=space_id, pr_num=pr_num, action="delete")
146
+
147
+
148
+ def notify_pr(
149
+ space_id: str, pr_num: int, action: Literal["create", "update", "delete"]
150
+ ) -> None:
151
+ ci_space_id = _get_ci_space_id(space_id=space_id, pr_num=pr_num)
152
+ if action == "create":
153
+ comment = NOTIFICATION_TEMPLATE_CREATE.format(ci_space_id=ci_space_id)
154
+ elif action == "update":
155
+ comment = NOTIFICATION_TEMPLATE_UPDATE.format(ci_space_id=ci_space_id)
156
+ elif action == "delete":
157
+ comment = NOTIFICATION_TEMPLATE_DELETE
158
+ else:
159
+ raise ValueError(f"Status {action} not handled.")
160
+
161
+ comment_discussion(
162
+ repo_id=space_id,
163
+ repo_type="space",
164
+ discussion_num=pr_num,
165
+ comment=comment,
166
+ token=HF_TOKEN,
167
+ )
168
+
169
+
170
+ def _get_ci_space_id(space_id: str, pr_num: int) -> str:
171
+ return f"{space_id}-ci-pr-{pr_num}"
172
+
173
+
174
+ NOTIFICATION_TEMPLATE_CREATE = """\
175
+ Hey there!
176
+ Following the creation of this PR, a temporary test Space [{ci_space_id}}](https://huggingface.co/spaces/{ci_space_id}) has been launched.
177
+ Any changes pushed to this PR will be synced with the test Space.
178
+
179
+ (This is an automated message)
180
+ """
181
+
182
+ NOTIFICATION_TEMPLATE_UPDATE = """\
183
+ Hey there!
184
+ Following new commits that happened in this PR, the temporary test Space [{ci_space_id}}](https://huggingface.co/spaces/{ci_space_id}) has been updated.
185
+
186
+ (This is an automated message)
187
+ """
188
+
189
+ NOTIFICATION_TEMPLATE_DELETE = """\
190
+ Hey there!
191
+ PR is now merged. The temporary test Space has been deleted.
192
+
193
+ (This is an automated message)
194
+ """
home.html ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!DOCTYPE html>
2
+ <html>
3
+ <head>
4
+ <meta charset="utf-8" />
5
+ <meta name="viewport" content="width=device-width" />
6
+ <title>Spaces CI bot</title>
7
+ <link rel="stylesheet" href="style.css" />
8
+ </head>
9
+ <body>
10
+ <div class="card">
11
+ <h1>Spaces CI webhook</h1>
12
+
13
+ <p>This is a webhook space to build temporary Spaces when a PR is submitted.</p>
14
+
15
+ <!-- <p>Check out the guide <a href="https://huggingface.co/docs/hub/webhooks-guide-auto-retrain" target="_blank">here</a>!</p> -->
16
+ </div>
17
+ </body>
18
+ </html>
requirements.txt ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ fastapi==0.74.*
2
+ requests==2.27.*
3
+ huggingface_hub==0.12.*
4
+ uvicorn[standard]==0.17.*