1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251 | """Factory class for creating dataset configurations."""
import logging
import sys
import typing as t
import torch
from .data_models import BenchmarkConfig, BenchmarkConfigParams
from .dataset_configs import get_all_dataset_configs
from .enums import Device
from .exceptions import InvalidBenchmark
from .languages import get_all_languages
from .tasks import SPEED, get_all_tasks
if t.TYPE_CHECKING:
from .data_models import Language, Task
logger = logging.getLogger("euroeval")
def build_benchmark_config(
benchmark_config_params: BenchmarkConfigParams,
) -> BenchmarkConfig:
"""Create a benchmark configuration.
Args:
benchmark_config_params:
The parameters for creating the benchmark configuration.
Returns:
The benchmark configuration.
"""
language_codes = get_correct_language_codes(
language_codes=benchmark_config_params.language
)
model_languages = prepare_languages(
language_codes=benchmark_config_params.model_language,
default_language_codes=language_codes,
)
dataset_languages = prepare_languages(
language_codes=benchmark_config_params.dataset_language,
default_language_codes=language_codes,
)
tasks, datasets = prepare_tasks_and_datasets(
task=benchmark_config_params.task,
dataset=benchmark_config_params.dataset,
dataset_languages=dataset_languages,
)
return BenchmarkConfig(
model_languages=model_languages,
dataset_languages=dataset_languages,
tasks=tasks,
datasets=datasets,
batch_size=benchmark_config_params.batch_size,
raise_errors=benchmark_config_params.raise_errors,
cache_dir=benchmark_config_params.cache_dir,
api_key=benchmark_config_params.api_key,
force=benchmark_config_params.force,
progress_bar=benchmark_config_params.progress_bar,
save_results=benchmark_config_params.save_results,
verbose=benchmark_config_params.verbose or benchmark_config_params.debug,
device=prepare_device(device=benchmark_config_params.device),
trust_remote_code=benchmark_config_params.trust_remote_code,
clear_model_cache=benchmark_config_params.clear_model_cache,
evaluate_test_split=benchmark_config_params.evaluate_test_split,
few_shot=benchmark_config_params.few_shot,
num_iterations=(
1
if hasattr(sys, "_called_from_test")
else benchmark_config_params.num_iterations
),
api_base=benchmark_config_params.api_base,
api_version=benchmark_config_params.api_version,
gpu_memory_utilization=benchmark_config_params.gpu_memory_utilization,
generative_type=benchmark_config_params.generative_type,
debug=benchmark_config_params.debug,
run_with_cli=benchmark_config_params.run_with_cli,
requires_safetensors=benchmark_config_params.requires_safetensors,
download_only=benchmark_config_params.download_only,
)
def get_correct_language_codes(language_codes: str | list[str]) -> list[str]:
"""Get correct language code(s).
Args:
language_codes:
The language codes of the languages to include, both for models and
datasets. Here 'no' means both Bokmål (nb) and Nynorsk (nn). Set this
to 'all' if all languages should be considered.
Returns:
The correct language codes.
"""
# Create a dictionary that maps languages to their associated language objects
language_mapping = get_all_languages()
# Create the list `languages`
if "all" in language_codes:
languages = list(language_mapping.keys())
elif isinstance(language_codes, str):
languages = [language_codes]
else:
languages = language_codes
# If `languages` contains 'no' then also include 'nb' and 'nn'. Conversely, if
# either 'nb' or 'nn' are specified then also include 'no'.
if "no" in languages:
languages = list(set(languages) | {"nb", "nn"})
elif "nb" in languages or "nn" in languages:
languages = list(set(languages) | {"no"})
return languages
def prepare_languages(
language_codes: str | list[str] | None, default_language_codes: list[str]
) -> list["Language"]:
"""Prepare language(s) for benchmarking.
Args:
language_codes:
The language codes of the languages to include for models or datasets.
If specified then this overrides the `language` parameter for model or
dataset languages.
default_language_codes:
The default language codes of the languages to include.
Returns:
The prepared dataset languages.
"""
# Create a dictionary that maps languages to their associated language objects
language_mapping = get_all_languages()
# Create the list `languages_str` of language codes to use for models or datasets
languages_str: list[str]
if language_codes is None:
languages_str = default_language_codes
elif isinstance(language_codes, str):
languages_str = [language_codes]
else:
languages_str = language_codes
# Convert the model languages to language objects
if "all" in languages_str:
prepared_languages = list(language_mapping.values())
else:
prepared_languages = [language_mapping[language] for language in languages_str]
return prepared_languages
def prepare_tasks_and_datasets(
task: str | list[str] | None,
dataset_languages: list["Language"],
dataset: str | list[str] | None,
) -> tuple[list["Task"], list[str]]:
"""Prepare task(s) and dataset(s) for benchmarking.
Args:
task:
The tasks to include for dataset. If None then datasets will not be
filtered based on their task.
dataset_languages:
The languages of the datasets in the benchmark.
dataset:
The datasets to include for task. If None then all datasets will be
included, limited by the `task` and `dataset_languages` parameters.
Returns:
The prepared tasks and datasets.
Raises:
InvalidBenchmark:
If the task or dataset is not found in the benchmark tasks or datasets.
"""
# Create a dictionary that maps benchmark tasks to their associated benchmark
# task objects, and a dictionary that maps dataset names to their associated
# dataset configuration objects
task_mapping = get_all_tasks()
all_dataset_configs = get_all_dataset_configs()
# Create the list of dataset tasks
try:
if task is None:
tasks = [t for t in task_mapping.values() if t != SPEED]
elif isinstance(task, str):
tasks = [task_mapping[task]]
else:
tasks = [task_mapping[t] for t in task]
except KeyError as e:
raise InvalidBenchmark(f"Task {e} not found in the benchmark tasks.") from e
all_official_datasets = [
dataset_name
for dataset_name, dataset_config in all_dataset_configs.items()
if not dataset_config.unofficial
]
if dataset is None:
dataset = all_official_datasets
elif isinstance(dataset, str):
dataset = [dataset]
all_datasets = list(all_dataset_configs.keys())
invalid_datasets = set(dataset) - set(all_datasets)
if invalid_datasets:
raise InvalidBenchmark(
f"Dataset(s) {', '.join(invalid_datasets)} not found in the benchmark "
"datasets."
)
datasets = [
dataset_name
for dataset_name, dataset_config in all_dataset_configs.items()
if dataset_name in dataset
and dataset_config.task in tasks
and set(dataset_config.languages).intersection(dataset_languages)
]
return tasks, datasets
def prepare_device(device: Device | None) -> torch.device:
"""Prepare device for benchmarking.
Args:
device:
The device to use for running the models. If None then the device will be
set automatically.
Returns:
The prepared device.
"""
device_mapping = {
Device.CPU: torch.device("cpu"),
Device.CUDA: torch.device("cuda"),
Device.MPS: torch.device("mps"),
}
if isinstance(device, Device):
return device_mapping[device]
if torch.cuda.is_available():
return torch.device("cuda")
elif torch.backends.mps.is_available():
return torch.device("mps")
else:
return torch.device("cpu")
|