File size: 19,600 Bytes
146b688 7d902d9 146b688 f7e9cda 9355489 ca60da9 146b688 40c29ba ace58da 146b688 ace58da b42419d 40c29ba 215d189 d7c0630 215d189 d7c0630 146b688 40c29ba 146b688 40c29ba 146b688 f7e9cda 40c29ba f7e9cda 146b688 9355489 146b688 f7e9cda 146b688 ca60da9 f7e9cda 146b688 f7e9cda 146b688 f7e9cda 146b688 40c29ba b42419d 891eeb1 40c29ba 891eeb1 40c29ba 891eeb1 1c7b1ad 891eeb1 b42419d ca60da9 b42419d 40c29ba b42419d ca60da9 b42419d ca60da9 b42419d 40c29ba ace58da 146b688 9355489 ace58da 40c29ba 3219568 ace58da 0467a8f ace58da 40c29ba fa96c3a 40c29ba ace58da f0948c9 ace58da 40c29ba 146b688 40c29ba 146b688 ca60da9 d7c0630 ca60da9 0467a8f ca60da9 40c29ba 3219568 40c29ba 1c7b1ad 3219568 40c29ba 1047c44 3219568 ace58da 40c29ba 3219568 40c29ba e0d1b89 d7c0630 e0d1b89 d7c0630 1047c44 d7c0630 e0d1b89 d7c0630 e0d1b89 40c29ba b42419d d7c0630 215d189 891eeb1 f76d0f5 5c952e8 f76d0f5 40c29ba d7c0630 40c29ba f76d0f5 40c29ba f76d0f5 d7c0630 f76d0f5 891eeb1 8ed231f 891eeb1 215d189 5c952e8 215d189 146b688 9355489 146b688 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 |
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Stick To Your Role! Leaderboard</title>
<!-- Include Bootstrap CSS for styling -->
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/5.1.3/css/bootstrap.min.css">
<!-- Include DataTables CSS -->
<link rel="stylesheet" href="https://cdn.datatables.net/1.11.5/css/dataTables.bootstrap5.min.css">
<!-- Custom CSS for additional styling -->
<style>
html, body {
height: 100%;
overflow: auto;
}
body {
background-color: #f8f9fa;
font-family: 'Arial', sans-serif;
}
.container {
max-width: 1200px; /* Limit the width of the container */
margin: auto; /* Center the container */
padding: 15px; /* Add some padding */
background: #fff;
border-radius: 8px;
box-shadow: 0 4px 8px rgba(0,0,0,0.1);
}
.container h1 {
color: #333;
text-align: center;
}
.container h3 {
margin-top: 10px;
margin-bottom: 50px;
margin-left: 20px;
margin-right: 20px;
text-align: center;
}
p {
margin: auto; /* Center the table */
margin-top: 20px;
margin-bottom: 10px;
max-width: 1000px; /* Adjust the width as needed */
text-align: left;
}
ul {
margin: auto; /* Center the table */
margin-top: 20px;
margin-bottom: 10px;
max-width: 1000px; /* Adjust the width as needed */
text-align: left;
list-style-type: disc;
padding-left: 20px; /* Add padding to indent list items */
}
ul li {
margin-bottom: 10px; /* Add space between list items */
}
.table-responsive {
margin-top: 20px;
max-width: 1000px; /* Adjust the width as needed */
margin: auto; /* Center the table */
}
.main-table {
font-size: 15px
}
.full-table {
font-size: 12px
}
table {
border-collapse: separate;
border-spacing: 0;
width: 1000px;
margin: auto;
border: none; /* Remove any default border */
}
table thead th {
background-color: #610b5d;
color: white;
border: 1px solid #dee2e6;
text-align: left;
}
table tbody tr {
background-color: #fff;
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
}
table tbody tr:hover {
background-color: #f1f1f1;
}
table td, table th {
padding: 5px; /* Reduce padding */
border: 1px solid #dee2e6;
}
table th:first-child {
border-top-left-radius: 10px;
}
table th:last-child {
border-top-right-radius: 10px;
}
.section {
padding-left: 150px;
padding-right: 150px;
text-align: left;
}
.citation-section {
margin-top: 5px;
text-align: center;
max-width: 1000px;
margin: auto;
}
.citation-box {
background-color: #f8f9fa;
border: 1px solid #dee2e6;
border-radius: 8px;
padding: 5px;
margin-top: 5px;
font-size: 13px;
text-align: left;
font-family: 'Courier New', Courier, monospace;
white-space: pre;
}
.image-container {
display: flex;
justify-content: center;
gap: 10px;
margin-bottom: 40px;
max-width: 1100px; /* Adjust the width as needed */
margin: auto;
}
.image-container a {
flex: 1;
}
.image-container img {
max-width: 100%;
height: auto;
display: block;
margin: auto;
}
.about-button {
text-align: center;
margin-top: 50px;
margin-bottom: 50px;
}
.custom-button {
background-color: #610b5d;
color: #fff; /* Set white text color */
border-radius: 15px; /* Rounded edges */
padding: 10px 20px; /* Padding for the button */
font-size: 18px; /* Increase font size */
text-decoration: none; /* Remove underline */
}
.custom-button:hover {
background-color: #812b7d;
color: #fff;
}
/* Hide default sorting arrows */
table.dataTable thead .sorting:after,
table.dataTable thead .sorting:before,
table.dataTable thead .sorting_asc:before,
table.dataTable thead .sorting_asc:after,
table.dataTable thead .sorting_desc:before,
table.dataTable thead .sorting_desc:after {
display: none;
}
table.dataTable thead .sorting_asc {
background-image: url("{{ url_for('static', filename='icons/sort_asc_gray.png') }}");
background-repeat: no-repeat;
background-position: center right;
}
table.dataTable thead .sorting_desc {
background-image: url("{{ url_for('static', filename='icons/sort_desc_gray.png') }}");
background-repeat: no-repeat;
background-position: center right;
}
/* Override DataTables padding for sorting icons */
table.dataTable > thead > tr > th:not(.sorting_disabled),
table.dataTable > thead > tr > td:not(.sorting_disabled) {
padding-right: 5px; /* Adjust as needed to center the text */
}
/* Align first two columns to the left */
table.dataTable tbody td:first-child,
table.dataTable tbody td:nth-child(2),
table.dataTable thead th:first-child,
table.dataTable thead th:nth-child(2) {
text-align: left;
}
/* Align all other columns to the center */
table.dataTable tbody td:not(:first-child):not(:nth-child(2)),
table.dataTable thead th:not(:first-child):not(:nth-child(2)) {
text-align: center;
}
/* Tooltip styling */
.tooltip-inner {
max-width: none;
}
th[title] {
position: relative;
cursor: help;
}
th[title]:hover::after {
content: attr(title);
position: absolute;
bottom: 100%;
left: 50%;
transform: translateX(-50%);
background-color: #333;
color: white;
padding: 5px 10px;
border-radius: 4px;
white-space: nowrap;
z-index: 1;
font-weight: normal;
font-size: 14px;
}
</style>
</head>
<body>
<div class="container">
<h1 class="mt-5">Stick To Your Role! Leaderboard</h1>
<h3>
LLMs can role-play different personas by simulating their values and behavior, but can they stick to their role whatever the context?
Is simulated Joan of Arc more tradition-driven than Elvis?
Will it still be the case after playing chess?
</h3>
<p>
The Stick to Your Role! leaderboard compares LLMs based on <b>undesired sensitivity to context change</b>.
LLM-exhibited behavior always depends on the context (prompt).
While some context-dependence is desired (e.g. following instructions),
some is undesired (e.g. drastically changing the simulated value expression based on the interlocutor).
As proposed in our <a target="_blank" href="https://arxiv.org/abs/2402.14846">paper</a>,
undesired context-dependence should be seen as a <b>property of LLMs</b> - a dimension of LLM comparison (alongside others such as model size speed or expressed knowledge).
This leaderboard aims to provide such a comparison and extends our paper with a more focused and elaborate experimental setup.
Standard benchmarks present <b>many</b> questions from the <b>same minimal contexts</b> (e.g. multiple choice questions),
we present <b>same</b> questions from <b>many different contexts</b>.
</p>
<p>
The Stick to You Role! leaderboard focuses on the <b>stability of simulated personal values during role-playing</b>.
We study the <b>coherence of a simulated population</b>.
In contrast to evaluating each simulated persona separately, we evaluate personas relative to each other, i.e. as a population.
You can browse the simulated population, questionnaires, and contexts used on our <a target="_blank" href="https://huggingface.co/datasets/flowers-team/StickToYourRole">🤗 StickToYourRole dataset</a>.
</p>
<div class="table-responsive main-table">
<!-- Render the table HTML here -->
{{ main_table_html|safe }}
</div>
<div class="image-container">
<a href="{{ url_for('static', filename='models_data/ordinal.svg') }}" target="_blank">
<img src="{{ url_for('static', filename='models_data/ordinal.svg') }}" alt="Ordinal">
</a>
<a href="{{ url_for('static', filename='models_data/cardinal.svg') }}" target="_blank">
<img src="{{ url_for('static', filename='models_data/cardinal.svg') }}" alt="Cardinal">
</a>
</div>
<p>
We leverage Schwartz's theory of <a href="https://www.sciencedirect.com/science/article/abs/pii/S0065260108602816">Basic Personal Values</a>,
which defines 10 values Self-Direction, Stimulation, Hedonism, Achievement, Power, Security, Conformity, Tradition, Benevolence, Universalism),
and the associated PVQ-40 and SVS questionnaires (available <a href="https://www.researchgate.net/publication/354384463_A_Repository_of_Schwartz_Value_Scales_with_Instructions_and_an_Introduction">here</a>).
</p>
<p>
Using the <a href="https://pubmed.ncbi.nlm.nih.gov/31402448/">methodology from psychology</a>, we focus on population-level (interpersonal) value stability, i.e. <b>Rank-Order stability (RO stability)</b>.
Rank-Order stability refers to the extent to which the order of different personas (in terms of expression of some value) remains the same along different contexts.
Refer <a href="{{ url_for('about', _anchor='rank_order_stability') }}">here</a> or to our <a href="https://arxiv.org/abs/2402.14846">paper</a> for more details.
</p>
<p>
In addition to Rank-Order stability we compute <b>validity metrics (Stress, CFI, SRMR, RMSEA)</b>, which are a common practice in psychology.
Validity refers to the extent to which the questionnaire measures what it purports to measure.
It can be seen as the questionnaire's accuracy in measuring the intended factors, i.e. values.
For example, basic personal values should be organized in a circular structure, and questions measuring the same value should be correlated.
The table below additionally shows the validity metrics, refer <a href="{{ url_for('about', _anchor='metrics') }}">here</a> for more details.
</p>
<p>
We <b>aggregate</b> Rank-Order stability and validation metrics to rank the models. We do so in two ways: <b>Cardinal</b> and <b>Ordinal</b>.
Following <a href="https://arxiv.org/abs/2405.01719">this paper</a>, we compute the stability and diversity of those rankings. See <a href="{{ url_for('about', _anchor='aggregate_metrics') }}">here</a> for more details.
</p>
<p>
To sum up here are the metrics used:
<ul>
<li><b>RO-stability</b> -
<i>Do the same simulated participants always (in every context) express same values more strongly than other participants?</i>
<!-- Do participants that express some value more than other participants express that value more in other contexts as well?-->
<!-- If a simulated participant expresses some value more strongly than another participant, will it express it more in other contexts as well?-->
<div style="margin-left: 20px; margin-top: 5px">
The correlation in the order of simulated participants (ordered based on the expression of the same values) over different contexts
</div>
</li>
<!--Validation metrics:-->
<li><b>Stress</b> -
<i>Is value expression (intercorrelations of values) structured as predicted by the theory (and as in humans), i.e. in a circle?</i>
<div style="margin-left: 20px; margin-top: 5px">
The Multi-dimensional scaling (MDS) fit of the observed value structure to the theoretical circular structure. Stress of 0 indicates 'perfect' fit, 0.025 excellent, 0.05 good, 0.1 fair, and 0.2 poor.
</div>
</li>
<!-- <li><b>Separability</b>: the extent to which questions corresponding to different values are linearly separable in the 2D MDS space (linear multi-label SVM classifier accuracy)</li>-->
<li><b>CFI, SRMR, RMSEA</b> -
<i>To what extent does the questionnaire measure what it is supposed to measure - values?</i>
<div style="margin-left: 20px; margin-top: 5px">
Common Confirmatory Factor Analysis (CFA) metrics showing the fit of the posited model of the relation of items (questions) to factors (values) on the observed data, applied here with Magnifying Glass CFA. For CFI >.90 is considered acceptable fit, for SRMR and RMSEA is <.05 considered good fit and <.08 reasonable.
</div>
</li>
<!--Aggregate metrics:-->
<li><b>Ordinal - Win Rate</b> -
<i>Which model beats the most other models across most metrics?</i>
<div style="margin-left: 20px; margin-top: 5px">
The score averaged over all metrics (with descending metrics inverted), context pairs (for stability) and contexts (for validity metrics)
<div>
</li>
<li><b>Cardinal - Score</b> -
<i>Which model has the highest average score?</i>
<div style="margin-left: 20px; margin-top: 5px">
The percentage of won games, where a game is a comparison of each model pair, each metric, and each context pair (for stability) or context (for validity metrics)
</div>
</li>
</ul>
</p>
<div class="table-responsive full-table">
<!-- Render the table HTML here -->
{{ full_table_html|safe }}
</div>
<div class="about-button">
<a href="{{ url_for('about') }}" class="custom-button mt-3">Motivation and Methodology page</a>
<a href="{{ url_for('new_model') }}" class="custom-button mt-3">Submit a model</a>
</div>
<div class="citation-section">
<p>
You can find more details in our <a target="_blank" href="https://arxiv.org/abs/2402.14846">paper</a>.
</p>
<p>
If you found this project useful, please cite one of our related papers,
which this leaderboard extends with a more focused and elaborate experimental setup.
Refer to the <a href="{{ url_for('about', _anchor='paper') }}">site</a> for details.
</p>
<p>Short paper: <a target="_blank" href="https://escholarship.org/uc/item/7w4823c6">Kovač, G., Portelas, R., Sawayama, M., Dominey, P. F., & Oudeyer, P. Y. (2024). Stick to your Role! Stability of Personal Values Expressed in Large Language Models. In Proceedings of the Annual Meeting of the Cognitive Science Society (Vol. 46).</a> </p>
<div class="citation-box" id="citation-text">
@inproceedings{kovavc2024stick,
title={Stick to your Role! Stability of Personal Values Expressed in Large Language Models},
author={Kova{\v{c}}, Grgur and Portelas, R{\'e}my and Sawayama, Masataka and Dominey, Peter Ford and Oudeyer, Pierre-Yves},
booktitle={Proceedings of the Annual Meeting of the Cognitive Science Society},
volume={46},
year={2024}
}
</div>
<p>Longer paper: <a target="_blank" href="https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0309114"> Kovač G, Portelas R, Sawayama M, Dominey PF, Oudeyer PY (2024) Stick to your role! Stability of personal values expressed in large language models. PLOS ONE 19(8): e0309114. https://doi.org/10.1371/journal.pone.0309114 </a></p>
<div class="citation-box" id="citation-text">
@article{kovavc2024stick,
title={Stick to your role! Stability of personal values expressed in large language models},
author={Kova{\v{c}}, Grgur and Portelas, R{\'e}my and Sawayama, Masataka and Dominey, Peter Ford and Oudeyer, Pierre-Yves},
journal={PloS one},
volume={19},
number={8},
pages={e0309114},
year={2024},
publisher={Public Library of Science San Francisco, CA USA}
}
</div>
</div>
<ul>
<li>Contact: <a href="mailto: [email protected]">[email protected]</a></li>
<li>See the <a target="_blank" href="https://sites.google.com/view/llmvaluestability">Project website</a></li>
<li>See the Flowers team <a target="_blank" href="http://developmentalsystems.org">blog</a> and <a target="_blank" href="https://flowers.inria.fr/">website</a></li>
<li>See Grgur's website and other projects: <a target="_blank" href="https://grgkovac.github.io">https://grgkovac.github.io</a></li>
</ul>
</div>
<!-- Include jQuery -->
<script src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
<!-- Include Bootstrap JS -->
<script src="https://stackpath.bootstrapcdn.com/bootstrap/5.1.3/js/bootstrap.bundle.min.js"></script>
<!-- Include DataTables JS -->
<script src="https://cdn.datatables.net/1.11.5/js/jquery.dataTables.min.js"></script>
<script src="https://cdn.datatables.net/1.11.5/js/dataTables.bootstrap5.min.js"></script>
<!-- Initialize DataTables -->
<script>
$(document).ready(function() {
const table = $('table').DataTable({
"paging": false,
"info": false,
"columnDefs": [
{ "orderable": false, "targets": 0 },
{ "searchable": false, "targets": 0 }
],
"order": [[ 2, 'desc' ]],
"drawCallback": function(settings) {
var api = this.api();
api.column(0, {order:'applied'}).nodes().each(function(cell, i) {
cell.innerHTML = i + 1;
});
}
});
});
</script>
</body>
</html>
|