File size: 16,297 Bytes
146b688 f7e9cda 9355489 ca60da9 146b688 40c29ba ace58da 146b688 ace58da b42419d 40c29ba 215d189 146b688 40c29ba 146b688 40c29ba 146b688 f7e9cda 40c29ba f7e9cda 146b688 9355489 146b688 f7e9cda 146b688 ca60da9 f7e9cda 146b688 f7e9cda 146b688 f7e9cda 146b688 40c29ba b42419d 891eeb1 40c29ba 891eeb1 40c29ba 891eeb1 1c7b1ad 891eeb1 b42419d ca60da9 b42419d 40c29ba b42419d ca60da9 b42419d ca60da9 b42419d 40c29ba ace58da 146b688 9355489 ace58da 40c29ba 3219568 ace58da 40c29ba ace58da 40c29ba fa96c3a 40c29ba ace58da 40c29ba 146b688 40c29ba 146b688 ca60da9 40c29ba 3219568 40c29ba 1c7b1ad 3219568 40c29ba 1047c44 3219568 ace58da 40c29ba 3219568 40c29ba e0d1b89 3219568 1047c44 ace58da e0d1b89 3219568 e0d1b89 40c29ba b42419d 215d189 891eeb1 40c29ba 891eeb1 1c7b1ad 891eeb1 1c7b1ad 891eeb1 215d189 ace58da 215d189 fa96c3a 215d189 146b688 9355489 146b688 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 |
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Stick To Your Role! Leaderboard</title>
<!-- Include Bootstrap CSS for styling -->
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/5.1.3/css/bootstrap.min.css">
<!-- Include DataTables CSS -->
<link rel="stylesheet" href="https://cdn.datatables.net/1.11.5/css/dataTables.bootstrap5.min.css">
<!-- Custom CSS for additional styling -->
<style>
body {
background-color: #f8f9fa;
font-family: 'Arial', sans-serif;
}
.container {
max-width: 1200px; /* Limit the width of the container */
margin: auto; /* Center the container */
padding: 15px; /* Add some padding */
background: #fff;
border-radius: 8px;
box-shadow: 0 4px 8px rgba(0,0,0,0.1);
}
.container h1 {
color: #333;
text-align: center;
}
.container h3 {
margin-top: 10px;
margin-bottom: 50px;
margin-left: 20px;
margin-right: 20px;
text-align: center;
}
p {
margin: auto; /* Center the table */
margin-top: 20px;
margin-bottom: 10px;
max-width: 1000px; /* Adjust the width as needed */
text-align: left;
}
ul {
margin: auto; /* Center the table */
margin-top: 20px;
margin-bottom: 10px;
max-width: 1000px; /* Adjust the width as needed */
text-align: left;
}
.table-responsive {
margin-top: 20px;
max-width: 1000px; /* Adjust the width as needed */
margin: auto; /* Center the table */
}
.main-table {
font-size: 15px
}
.full-table {
font-size: 12px
}
table {
border-collapse: separate;
border-spacing: 0;
width: 1000px;
margin: auto;
border: none; /* Remove any default border */
}
table thead th {
background-color: #610b5d;
color: white;
border: 1px solid #dee2e6;
text-align: left;
}
table tbody tr {
background-color: #fff;
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
}
table tbody tr:hover {
background-color: #f1f1f1;
}
table td, table th {
padding: 5px; /* Reduce padding */
border: 1px solid #dee2e6;
}
table th:first-child {
border-top-left-radius: 10px;
}
table th:last-child {
border-top-right-radius: 10px;
}
.section {
padding-left: 150px;
padding-right: 150px;
text-align: left;
}
.citation-section {
margin-top: 5px;
text-align: center;
max-width: 1000px;
margin: auto;
}
.citation-box {
background-color: #f8f9fa;
border: 1px solid #dee2e6;
border-radius: 8px;
padding: 5px;
margin-top: 5px;
font-size: 13px;
text-align: left;
font-family: 'Courier New', Courier, monospace;
white-space: pre;
}
.image-container {
display: flex;
justify-content: center;
gap: 10px;
margin-bottom: 40px;
max-width: 1100px; /* Adjust the width as needed */
margin: auto;
}
.image-container a {
flex: 1;
}
.image-container img {
max-width: 100%;
height: auto;
display: block;
margin: auto;
}
.about-button {
text-align: center;
margin-top: 50px;
margin-bottom: 50px;
}
.custom-button {
background-color: #610b5d;
color: #fff; /* Set white text color */
border-radius: 15px; /* Rounded edges */
padding: 10px 20px; /* Padding for the button */
font-size: 18px; /* Increase font size */
text-decoration: none; /* Remove underline */
}
.custom-button:hover {
background-color: #812b7d;
color: #fff;
}
/* Hide default sorting arrows */
table.dataTable thead .sorting:after,
table.dataTable thead .sorting:before,
table.dataTable thead .sorting_asc:before,
table.dataTable thead .sorting_asc:after,
table.dataTable thead .sorting_desc:before,
table.dataTable thead .sorting_desc:after {
display: none;
}
table.dataTable thead .sorting_asc {
background-image: url("{{ url_for('static', filename='icons/sort_asc_gray.png') }}");
background-repeat: no-repeat;
background-position: center right;
}
table.dataTable thead .sorting_desc {
background-image: url("{{ url_for('static', filename='icons/sort_desc_gray.png') }}");
background-repeat: no-repeat;
background-position: center right;
}
/* Override DataTables padding for sorting icons */
table.dataTable > thead > tr > th:not(.sorting_disabled),
table.dataTable > thead > tr > td:not(.sorting_disabled) {
padding-right: 5px; /* Adjust as needed to center the text */
}
/* Align first two columns to the left */
table.dataTable tbody td:first-child,
table.dataTable tbody td:nth-child(2),
table.dataTable thead th:first-child,
table.dataTable thead th:nth-child(2) {
text-align: left;
}
/* Align all other columns to the center */
table.dataTable tbody td:not(:first-child):not(:nth-child(2)),
table.dataTable thead th:not(:first-child):not(:nth-child(2)) {
text-align: center;
}
/* Tooltip styling */
.tooltip-inner {
max-width: none;
}
th[title] {
position: relative;
cursor: help;
}
th[title]:hover::after {
content: attr(title);
position: absolute;
bottom: 100%;
left: 50%;
transform: translateX(-50%);
background-color: #333;
color: white;
padding: 5px 10px;
border-radius: 4px;
white-space: nowrap;
z-index: 1;
font-weight: normal;
font-size: 14px;
}
</style>
</head>
<body>
<div class="container">
<h1 class="mt-5">Stick To Your Role! Leaderboard</h1>
<h3>
LLMs can role-play different personas by simulating their values and behavior, but can they stick to their role whatever the context?
Is simulated Joan of Arc more tradition-driven than Elvis?
Will it still be the case after playing chess?
</h3>
<p>
The Stick to Your Role! leaderboard compares LLMs based on <b>undesired sensitivity to context change</b>.
LLM-exhibited behavior always depends on the context (prompt).
While some context-dependence is desired (e.g. following instructions),
some is undesired (e.g. drastically changing the simulated value expression based on the interlocutor).
As proposed in our <a href="https://arxiv.org/abs/2402.14846">paper</a>,
undesired context-dependence should be seen as a <b>property of LLMs</b> - a dimension of LLM comparison (alongside others such as model size speed or expressed knowledge).
This leaderboard aims to provide such a comparison and extends our paper with a more focused and elaborate experimental setup.
Standard benchmarks present <b>many</b> questions from the <b>same minimal contexts</b> (e.g. multiple choice questions),
we present <b>same</b> questions from <b>many different contexts</b>.
</p>
<p>
The Stick to You Role! leaderboard focuses on the <b>stability of simulated personal values during role-playing</b>.
We study the <b>coherence of a simulated population</b>.
In contrast to evaluating each simulated persona separately, we evaluate personas relative to each other, i.e. as a population.
</p>
<div class="table-responsive main-table">
<!-- Render the table HTML here -->
{{ main_table_html|safe }}
</div>
<div class="image-container">
<a href="{{ url_for('static', filename='figures/cardinal.svg') }}" target="_blank">
<img src="{{ url_for('static', filename='figures/cardinal.svg') }}" alt="Cardinal">
</a>
<a href="{{ url_for('static', filename='figures/ordinal.svg') }}" target="_blank">
<img src="{{ url_for('static', filename='figures/ordinal.svg') }}" alt="Ordinal">
</a>
</div>
<p>
We leverage Schwartz's theory of <a href="https://www.sciencedirect.com/science/article/abs/pii/S0065260108602816">Basic Personal Values</a>,
which defines 10 values Self-Direction, Stimulation, Hedonism, Achievement, Power, Security, Conformity, Tradition, Benevolence, Universalism),
and the associated PVQ-40 and SVS questionnaires (available <a href="https://www.researchgate.net/publication/354384463_A_Repository_of_Schwartz_Value_Scales_with_Instructions_and_an_Introduction">here</a>).
</p>
<p>
Using the <a href="https://pubmed.ncbi.nlm.nih.gov/31402448/">methodology from psychology</a>, we focus on population-level (interpersonal) value stability, i.e. <b>Rank-Order stability (RO stability)</b>.
Rank-Order stability refers to the extent to which the order of different personas (in terms of expression of some value) remains the same along different contexts.
Refer <a href="{{ url_for('about', _anchor='rank_order_stability') }}">here</a> or to our <a href="https://arxiv.org/abs/2402.14846">paper</a> for more details.
</p>
<p>
<!-- In addition to Rank-Order stability we compute <b>validity metrics (Stress, Separability, CFI, SRMR, RMSEA)</b>, which are a common practice in psychology.-->
In addition to Rank-Order stability we compute <b>validity metrics (Stress, CFI, SRMR, RMSEA)</b>, which are a common practice in psychology.
Validity refers to the extent to which the questionnaire measures what it purports to measure.
It can be seen as the questionnaire's accuracy in measuring the intended factors, i.e. values.
For example, basic personal values should be organized in a circular structure, and questions measuring the same value should be correlated.
The table below additionally shows the validity metrics, refer <a href="{{ url_for('about', _anchor='metrics') }}">here</a> for more details.
</p>
<p>
We <b>aggregate</b> Rank-Order stability and validation metrics to rank the models. We do so in two ways: <b>Cardinal</b> and <b>Ordinal</b>.
Following <a href="https://arxiv.org/abs/2405.01719">this paper</a>, we compute the stability and diversity of those rankings. See <a href="{{ url_for('about', _anchor='aggregate_metrics') }}">here</a> for more details.
</p>
<p>
To sum up here are the metrics used:
<ul>
<li><b>RO-stability</b>: the correlation in the order of simulated participants (ordered based on the expression of the same values) over different contexts</li>
<!--Validation metrics:-->
<li><b>Stress</b>: the Multi-dimensional scaling (MDS) fit of the observed value structure to the theoretical circular structure. Stress of 0 indicates 'perfect' fit, 0.025 excellent, 0.05 good, 0.1 fair, and 0.2 poor.</li>
<!-- <li><b>Separability</b>: the extent to which questions corresponding to different values are linearly separable in the 2D MDS space (linear multi-label SVM classifier accuracy)</li>-->
<li><b>CFI, SRMR, RMSEA</b>: Common Confirmatory Factor Analysis (CFA) metrics showing the fit of the posited model of the relation of items (questions) to factors (values) on the observed data, applied here with Magnifying Glass CFA. For CFI >.90 is considered acceptable fit, for SRMR and RMSEA is <.05 considered good fit and <.08 reasonable.</li>
<!--Aggregate metrics:-->
<li><b>Ordinal - Win Rate</b>: the score averaged over all metrics (with descending metrics inverted), context pairs (for stability) and contexts (for validity metrics)</li>
<li><b>Cardinal - Score</b>: the percentage of won games, where a game is a comparison of each model pair, each metric, and each context pair (for stability) or context (for validity metrics)</li>
</ul>
</p>
<div class="table-responsive full-table">
<!-- Render the table HTML here -->
{{ full_table_html|safe }}
</div>
<div class="about-button">
<a href="{{ url_for('about') }}" class="custom-button mt-3">Learn More About This Project</a>
</div>
<div class="about-button">
<a href="{{ url_for('new_model') }}" class="custom-button mt-3">Submit a model</a>
</div>
<div class="citation-section">
<p>
If you found this project useful, please cite our related paper,
which this leaderboard extends with a more focused and elaborate experimental setup.
Refer <a href="{{ url_for('about', _anchor='paper') }}">here</a> for details.
</p>
<div class="citation-box" id="citation-text">
@inproceedings{kovavc2024stick,
title={Stick to your Role! Stability of Personal Values Expressed in Large Language Models},
author={Kova{\v{c}}, Grgur and Portelas, R{\'e}my and Sawayama, Masataka and Dominey, Peter Ford and Oudeyer, Pierre-Yves},
booktitle={Proceedings of the Annual Meeting of the Cognitive Science Society},
volume={46},
year={2024}
}
</div>
</div>
<ul>
<li>Contact: <a href="mailto: [email protected]">[email protected]</a></li>
<li>See the <a href="https://sites.google.com/view/llmvaluestability">Project website</a></li>
<li>See the Flowers team <a href="http://developmentalsystems.org">blog</a> and <a href="https://flowers.inria.fr/">website</a></li>
<li>See Grgur's website and other projects: <a href="https://grgkovac.github.io">https://grgkovac.github.io</a></li>
</ul>
</div>
<!-- Include jQuery -->
<script src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
<!-- Include Bootstrap JS -->
<script src="https://stackpath.bootstrapcdn.com/bootstrap/5.1.3/js/bootstrap.bundle.min.js"></script>
<!-- Include DataTables JS -->
<script src="https://cdn.datatables.net/1.11.5/js/jquery.dataTables.min.js"></script>
<script src="https://cdn.datatables.net/1.11.5/js/dataTables.bootstrap5.min.js"></script>
<!-- Initialize DataTables -->
<script>
$(document).ready(function() {
const table = $('table').DataTable({
"paging": false,
"info": false,
"columnDefs": [
{ "orderable": false, "targets": 0 },
{ "searchable": false, "targets": 0 }
],
"order": [[ 2, 'desc' ]],
"drawCallback": function(settings) {
var api = this.api();
api.column(0, {order:'applied'}).nodes().each(function(cell, i) {
cell.innerHTML = i + 1;
});
}
});
});
</script>
</body>
</html>
|