monitor/inspect_conv.py

import argparse
import code
import datetime
import json
import os
from pytz import timezone
import time

import pandas as pd
from tqdm import tqdm


def get_log_files(max_num_files=None):
    dates = []
    for month in [4, 5]:
        for day in range(1, 32):
            dates.append(f"2023-{month:02d}-{day:02d}")

    num_servers = 14
    filenames = []
    for d in dates:
        for i in range(num_servers):
            name = os.path.expanduser(f"~/fastchat_logs/server{i}/{d}-conv.json")
            if os.path.exists(name):
                filenames.append(name)
    max_num_files = max_num_files or len(filenames)
    filenames = filenames[-max_num_files:]
    return filenames


def pretty_print_conversation(messages):
    for role, msg in messages:
        print(f"[[{role}]]: {msg}")


def inspect_convs(log_files):
    data = []
    for filename in tqdm(log_files, desc="read files"):
        for retry in range(5):
            try:
                lines = open(filename).readlines()
                break
            except FileNotFoundError:
                time.sleep(2)

        for l in lines:
            row = json.loads(l)

            if "states" not in row:
                continue
            if row["type"] not in ["leftvote", "rightvote", "bothbad_vote"]:
                continue

            model_names = row["states"][0]["model_name"], row["states"][1]["model_name"]
            if row["type"] == "leftvote":
                winner, loser = model_names[0], model_names[1]
                winner_conv, loser_conv = row["states"][0], row["states"][1]
            elif row["type"] == "rightvote":
                loser, winner = model_names[0], model_names[1]
                loser_conv, winner_conv = row["states"][0], row["states"][1]

            if loser == "bard" and winner == "vicuna-13b":
                print("=" * 20)
                print(f"Winner: {winner}")
                pretty_print_conversation(winner_conv["messages"])
                print(f"Loser: {loser}")
                pretty_print_conversation(loser_conv["messages"])
                print("=" * 20)
                input()

            # if row["type"] == "bothbad_vote" and "gpt-4" in model_names:
            #    print("=" * 20)
            #    print(f"Model A: {model_names[0]}")
            #    pretty_print_conversation(row["states"][0]["messages"])
            #    print(f"Model B: {model_names[1]}")
            #    pretty_print_conversation(row["states"][1]["messages"])
            #    print("=" * 20)
            #    input()


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--max-num-files", type=int)
    args = parser.parse_args()

    log_files = get_log_files(args.max_num_files)
    inspect_convs(log_files)