Fix async bug and add readme

app.py

@@ -8,6 +8,7 @@ from src.my_logger import setup_logger
 from src.utilities import load_datasets, merge_and_update_datasets
 from src.visualize_logs import log_file_to_html_string
 from src.build_nomic import build_nomic
+from src.readme_update import update_dataset_readme
 
 proj_dir = Path(__name__).parent
 
@@ -96,13 +97,17 @@ async def community(payload: WebhookPayload):
     logger.info(f"Loaded new dataset")
 
     logger.info(f"Merging and Updating row...")
-    dataset = merge_and_update_datasets(dataset, original_dataset)
+    dataset, updated_row_count = merge_and_update_datasets(dataset, original_dataset)
 
     # Push the augmented dataset to the Hugging Face hub
     logger.info(f"Pushing processed data to the Hugging Face Hub...")
     dataset.push_to_hub(PROCESSED_DATASET, token=HUGGINGFACE_AUTH_TOKEN)
     logger.info(f"Pushed processed data to the Hugging Face Hub")
 
+    update_dataset_readme(dataset_name=PROCESSED_DATASET, subreddit=SUBREDDIT, new_rows=updated_row_count)
+    logger.info(f"Updated README.")
+
+    # Build Nomic
     logger.info(f"Building Nomic...")
     build_nomic(dataset=dataset)
     logger.info(f"Built Nomic")

src/readme_update.py

@@ -0,0 +1,93 @@
+import os
+from datetime import datetime
+from pathlib import Path
+from shutil import rmtree
+
+import pytz
+from huggingface_hub import HfApi, Repository
+
+GENERATED_BELOW_MARKER = "--- Generated Part of README Below ---"
+hf_token = os.environ["HUGGINGFACE_AUTH_TOKEN"]
+local_repo_path = "./readme_repo"
+
+
+def update_dataset_readme(dataset_name: str, subreddit: str, new_rows: int) -> None:
+    """
+    Update the README file of a specified dataset repository with new information.
+
+    Args:
+    dataset_name (str): Name of the dataset repository.
+    subreddit (str): Name of the subreddit being used for dataset creation.
+    new_rows (int): Number of new rows added in the latest update.
+    hf_token (str): Hugging Face authentication token.
+    local_repo_path (str): Local path to clone the repository.
+    """
+    # Initialize HfApi
+    api = HfApi()
+
+    if Path(local_repo_path).exists():
+        rmtree(local_repo_path)
+
+    # Clone the repository locally
+    repo = Repository(local_repo_path, clone_from=dataset_name, repo_type='dataset', use_auth_token=hf_token)
+
+    # Read the README file
+    with open(f"{local_repo_path}/README.md", "r") as file:
+        old_readme = file.read()
+
+    # Modify the README
+    new_readme = append_to_readme(subreddit=subreddit, new_rows=new_rows, old_readme=old_readme)
+
+    # Write the updated README back to the repository
+    with open(f"{local_repo_path}/README.md", "w") as file:
+        file.write(new_readme)
+
+    # Push the changes
+    repo.push_to_hub(blocking=True, commit_message=f'Pushing {new_rows} new rows')
+
+
+def append_to_readme(subreddit: str, new_rows: int, old_readme: str) -> str:
+    """
+    Append new information to the existing README content.
+
+    Args:
+    subreddit (str): Name of the subreddit.
+    new_rows (int): Number of new rows added.
+    old_readme (str): Existing README content.
+
+    Returns:
+    str: Updated README content.
+    """
+    latest_hour = datetime.now(pytz.utc).replace(minute=0, second=0, microsecond=0)
+    latest_hour_str = latest_hour.strftime('%Y-%m-%d %H:00:00 %Z%z')
+
+    readme_text = f"""
+## Dataset Overview
+This dataset is based on [derek-thomas/dataset-creator-reddit-{subreddit}](https://huggingface.co/datasets/derek-thomas/dataset-creator-reddit-{subreddit}) 
+and will add [nomic-ai/nomic-embed-text-v1](https://huggingface.co/nomic-ai/nomic-embed-text-v1) embeddings based on the
+`content` field.
+
+The goal is to be able to have an automatic and free semantic/neural tool for any subreddit.
+
+The last run was on {latest_hour_str} and updated {new_rows}.
+
+## Creation Details
+This is done by triggering [derek-thomas/processing-bestofredditorupdates](https://huggingface.co/spaces/derek-thomas/processing-bestofredditorupdates) 
+based on a repository update webhook to calculate the embeddings and update the [nomic atlas](https://docs.nomic.ai) 
+visualization. 
+
+## Update Frequency
+The dataset is updated based on a webhook trigger, so each time [derek-thomas/dataset-creator-reddit-{subreddit}](https://huggingface.co/datasets/derek-thomas/dataset-creator-reddit-{subreddit})
+is updated, this dataset will be updated. 
+
+## Opt-out
+To opt-out of this dataset please make a request in the community tab
+"""
+
+    if GENERATED_BELOW_MARKER in old_readme:
+        index = old_readme.index(GENERATED_BELOW_MARKER) + len(GENERATED_BELOW_MARKER)
+        new_readme = old_readme[:index] + "\n\n" + readme_text
+    else:
+        new_readme = old_readme + "\n\n" + GENERATED_BELOW_MARKER + "\n\n" + readme_text + "\n"
+
+    return new_readme

src/utilities.py

@@ -15,7 +15,7 @@ PROCESSED_DATASET = os.environ['PROCESSED_DATASET']
 logger = setup_logger(__name__)
 
 
-async def load_datasets():
+def load_datasets():
     # Get latest datasets locally
     logger.debug(f"Trying to download {PROCESSED_DATASET}")
     dataset = load_dataset(PROCESSED_DATASET, download_mode=DownloadMode.FORCE_REDOWNLOAD)
@@ -38,7 +38,7 @@ def merge_and_update_datasets(dataset, original_dataset):
     # Step 1: Merge df onto odf
     # We'll bring in 'content' and 'embedding' from df to compare and possibly update 'embedding'
     merged_df = pd.merge(odf, df[['id', 'content', 'embedding']], on='id', how='left', suffixes=('_odf', ''))
-    updated_rows = len(merged_df[merged_df.content != merged_df.content_odf])
+    updated_row_count = len(merged_df[merged_df.content != merged_df.content_odf])
 
     # Step 2: Compare 'content' from odf and df, update 'embedding' if they differ
     merged_df['embedding'] = np.where(merged_df['content_odf'] != merged_df['content'], None, merged_df['embedding'])
@@ -48,15 +48,15 @@ def merge_and_update_datasets(dataset, original_dataset):
     merged_df = merged_df.drop(columns=['content', 'new', 'updated'])  # Update columns to match df
     merged_df.rename(columns={'content_odf': 'content'}, inplace=True)  # Rename 'content_odf' back to 'content'
 
-    logger.info(f"Updating {updated_rows} rows...")
+    logger.info(f"Updating {updated_row_count} rows...")
     # Iterate over the DataFrame rows where 'embedding' is None
     for index, row in merged_df[merged_df['embedding'].isnull()].iterrows():
         # Update 'embedding' for the current row using our function
         merged_df.at[index, 'embedding'] = update_embeddings(content=row['content'], client=client)
 
     dataset['train'] = Dataset.from_pandas(merged_df)
-    logger.info(f"Updated {updated_rows} rows")
-    return dataset
+    logger.info(f"Updated {updated_row_count} rows")
+    return dataset, updated_row_count
 
 
 def update_embeddings(content, client):