File size: 11,958 Bytes
146b688 f7e9cda 9355489 ca60da9 146b688 40c29ba 146b688 b42419d 40c29ba 146b688 40c29ba 146b688 40c29ba 146b688 f7e9cda 40c29ba f7e9cda 146b688 9355489 146b688 f7e9cda 146b688 ca60da9 f7e9cda 146b688 f7e9cda 146b688 f7e9cda 146b688 40c29ba b42419d 891eeb1 40c29ba 891eeb1 40c29ba 891eeb1 40c29ba 891eeb1 b42419d ca60da9 b42419d 40c29ba b42419d ca60da9 b42419d ca60da9 b42419d 40c29ba 146b688 9355489 40c29ba 146b688 40c29ba 146b688 ca60da9 40c29ba b42419d 891eeb1 40c29ba 891eeb1 146b688 9355489 146b688 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 |
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Stick To Your Role! Leaderboard</title>
<!-- Include Bootstrap CSS for styling -->
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/5.1.3/css/bootstrap.min.css">
<!-- Include DataTables CSS -->
<link rel="stylesheet" href="https://cdn.datatables.net/1.11.5/css/dataTables.bootstrap5.min.css">
<!-- Custom CSS for additional styling -->
<style>
body {
background-color: #f8f9fa;
font-family: 'Arial', sans-serif;
}
.container {
max-width: 1200px; /* Limit the width of the container */
margin: auto; /* Center the container */
padding: 15px; /* Add some padding */
background: #fff;
border-radius: 8px;
box-shadow: 0 4px 8px rgba(0,0,0,0.1);
}
h1 {
color: #333;
text-align: center;
}
h2 {
color: #333;
margin-top: 30px;
text-align: center;
}
p {
margin: auto; /* Center the table */
margin-top: 20px;
margin-bottom: 10px;
max-width: 1000px; /* Adjust the width as needed */
text-align: left;
}
.table-responsive {
margin-top: 20px;
max-width: 1000px; /* Adjust the width as needed */
margin: auto; /* Center the table */
}
.main-table {
font-size: 15px
}
.full-table {
font-size: 12px
}
table {
border-collapse: separate;
border-spacing: 0;
width: 1000px;
margin: auto;
border: none; /* Remove any default border */
}
table thead th {
background-color: #610b5d;
color: white;
border: 1px solid #dee2e6;
text-align: left;
}
table tbody tr {
background-color: #fff;
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
}
table tbody tr:hover {
background-color: #f1f1f1;
}
table td, table th {
padding: 5px; /* Reduce padding */
border: 1px solid #dee2e6;
}
table th:first-child {
border-top-left-radius: 10px;
}
table th:last-child {
border-top-right-radius: 10px;
}
.section {
padding-left: 150px;
padding-right: 150px;
text-align: left;
}
.citation-section {
margin-top: 5px;
text-align: center;
max-width: 1000px;
margin: auto;
}
.citation-box {
background-color: #f8f9fa;
border: 1px solid #dee2e6;
border-radius: 8px;
padding: 5px;
margin-top: 5px;
font-size: 12px;
text-align: left;
font-family: 'Courier New', Courier, monospace;
white-space: pre;
}
.image-container {
display: flex;
justify-content: center;
gap: 10px;
margin-bottom: 40px;
max-width: 1100px; /* Adjust the width as needed */
margin: auto;
}
.image-container a {
flex: 1;
}
.image-container img {
max-width: 100%;
height: auto;
display: block;
margin: auto;
}
.about-button {
text-align: center;
margin-top: 50px;
margin-bottom: 50px;
}
.custom-button {
background-color: #610b5d;
color: #fff; /* Set white text color */
border-radius: 15px; /* Rounded edges */
padding: 10px 20px; /* Padding for the button */
font-size: 18px; /* Increase font size */
text-decoration: none; /* Remove underline */
}
.custom-button:hover {
background-color: #812b7d;
color: #fff;
}
/* Hide default sorting arrows */
table.dataTable thead .sorting:after,
table.dataTable thead .sorting:before,
table.dataTable thead .sorting_asc:before,
table.dataTable thead .sorting_asc:after,
table.dataTable thead .sorting_desc:before,
table.dataTable thead .sorting_desc:after {
display: none;
}
table.dataTable thead .sorting_asc {
background-image: url("{{ url_for('static', filename='icons/sort_asc_gray.png') }}");
background-repeat: no-repeat;
background-position: center right;
}
table.dataTable thead .sorting_desc {
background-image: url("{{ url_for('static', filename='icons/sort_desc_gray.png') }}");
background-repeat: no-repeat;
background-position: center right;
}
/* Override DataTables padding for sorting icons */
table.dataTable > thead > tr > th:not(.sorting_disabled),
table.dataTable > thead > tr > td:not(.sorting_disabled) {
padding-right: 5px; /* Adjust as needed to center the text */
}
/* Align first two columns to the left */
table.dataTable tbody td:first-child,
table.dataTable tbody td:nth-child(2),
table.dataTable thead th:first-child,
table.dataTable thead th:nth-child(2) {
text-align: left;
}
/* Align all other columns to the center */
table.dataTable tbody td:not(:first-child):not(:nth-child(2)),
table.dataTable thead th:not(:first-child):not(:nth-child(2)) {
text-align: center;
}
</style>
</head>
<body>
<div class="container">
<h1 class="mt-5">Stick To Your Role! Leaderboard</h1>
<p>
The Stick to Your Role! leaderboard compares LLMs based on <b>undesired sensitivity to context change</b>.
It focuses on the stability of personal value expression in simulated personas.
As proposed in our <a href="https://arxiv.org/abs/2402.14846">paper</a>,
unwanted context-dependence should be seen as a <b>property of LLMs</b> - a dimension of LLM comparison (alongside others such as model size speed or expressed knowledge).
This leaderboard aims to provide such a comparison and extends our paper with a more focused and elaborate experimental setup.
Standard benchmarks present MANY questions from the SAME MINIMAL contexts (e.g. multiple choice questions),
we present SAME questions from MANY different contexts.
</p>
<div class="table-responsive main-table">
<!-- Render the table HTML here -->
{{ main_table_html|safe }}
</div>
<div class="image-container">
<a href="{{ url_for('static', filename='figures/cardinal.svg') }}" target="_blank">
<img src="{{ url_for('static', filename='figures/cardinal.svg') }}" alt="Cardinal">
</a>
<a href="{{ url_for('static', filename='figures/ordinal.svg') }}" target="_blank">
<img src="{{ url_for('static', filename='figures/ordinal.svg') }}" alt="Ordinal">
</a>
</div>
<p>
We leverage the Schwartz's theory of <a href="https://www.sciencedirect.com/science/article/abs/pii/S0065260108602816">Basic Personal Values</a>,
which defines 10 values Self-Direction, Stimulation, Hedonism, Achievement, Power, Security, Conformity, Tradition, Benevolence, Universalism),
and the associated PVQ-40 and SVS questionnaires (available <a href="https://www.researchgate.net/publication/354384463_A_Repository_of_Schwartz_Value_Scales_with_Instructions_and_an_Introduction">here</a>).
</p>
<p>
Using the <a href="https://pubmed.ncbi.nlm.nih.gov/31402448/">methodology from psychology</a>, we focus on population-level (interpersonal) value stability, i.e. <b>Rank-Order stability</b> (RO stability).
Rank-Order stability refers to the extent the order of different personas (in terms of expression of some value) remains the same along different contexts.
Refer <a href="{{ url_for('about', _anchor='rank_order_stability') }}">here</a> or to our <a href="https://arxiv.org/abs/2402.14846">paper</a> for more details.
</p>
<p>
In addition to Rank-Order stability we compute <b>validity metrics</b>, which are a common practice in psychology.
Validity refers to the extent the questionnaire measures what it purports to measure.
It can be seen the questionnaire's accuracy in measuring the intended factors, i.e. values.
For example, basic personal values should be organized in a circular structure, and questions measuring the same value should be correlated.
The table below additionally shows the validity metrics, refer <a href="{{ url_for('about', _anchor='metrics') }}">here</a> for more details.
</p>
<p>
We <b>aggregate</b> Rank-Order stability and validation metrics to rank the models. We do so in two ways: <b>Cardinal</b> and <b>Ordinal</b>.
Following, <a href="https://arxiv.org/abs/2405.01719">this paper</a>, we compute the stability and diversity of those rankings. See <a href="{{ url_for('about', _anchor='aggregate_metrics') }}">here</a> for more details.
</p>
<div class="table-responsive full-table">
<!-- Render the table HTML here -->
{{ full_table_html|safe }}
</div>
<div class="about-button">
<a href="{{ url_for('about') }}" class="custom-button mt-3">Learn More About This Project</a>
</div>
<div class="citation-section">
<p>
If you found this project useful, please cite our related paper,
which this leaderboard extends with a more focused and elaborate experimental setup.
Refer <a href="{{ url_for('about', _anchor='paper') }}">here</a> for details.
</p>
<div class="citation-box" id="citation-text">
@article{kovavc2024stick,
title={Stick to your Role! Stability of Personal Values Expressed in Large Language Models},
author={Kova{\v{c}}, Grgur and Portelas, R{\'e}my and Sawayama, Masataka and Dominey, Peter Ford and Oudeyer, Pierre-Yves},
journal={arXiv preprint arXiv:2402.14846},
year={2024}
}
</div>
</div>
</div>
<!-- Include jQuery -->
<script src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
<!-- Include Bootstrap JS -->
<script src="https://stackpath.bootstrapcdn.com/bootstrap/5.1.3/js/bootstrap.bundle.min.js"></script>
<!-- Include DataTables JS -->
<script src="https://cdn.datatables.net/1.11.5/js/jquery.dataTables.min.js"></script>
<script src="https://cdn.datatables.net/1.11.5/js/dataTables.bootstrap5.min.js"></script>
<!-- Initialize DataTables -->
<script>
$(document).ready(function() {
const table = $('table').DataTable({
"paging": false,
"info": false,
"columnDefs": [
{ "orderable": false, "targets": 0 },
{ "searchable": false, "targets": 0 }
],
"order": [[ 2, 'desc' ]],
"drawCallback": function(settings) {
var api = this.api();
api.column(0, {order:'applied'}).nodes().each(function(cell, i) {
cell.innerHTML = i + 1;
});
}
});
});
</script>
</body>
</html>
|