1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178 | """All Norwegian dataset configurations used in EuroEval."""
from ..data_models import DatasetConfig
from ..languages import NB, NN, NO
from ..tasks import COMMON_SENSE, KNOW, LA, MCRC, NER, RC, SENT, SUMM
### Official datasets ###
NOREC_CONFIG = DatasetConfig(
name="norec",
pretty_name="the truncated version of the Norwegian sentiment classification "
"dataset NoReC",
huggingface_id="EuroEval/norec-mini",
task=SENT,
languages=[NB, NN, NO],
)
SCALA_NB_CONFIG = DatasetConfig(
name="scala-nb",
pretty_name="the Bokmål part of the linguistic acceptability dataset ScaLA",
huggingface_id="EuroEval/scala-nb",
task=LA,
languages=[NB, NO],
)
SCALA_NN_CONFIG = DatasetConfig(
name="scala-nn",
pretty_name="the Nynorsk part of the linguistic acceptability dataset ScaLA",
huggingface_id="EuroEval/scala-nn",
task=LA,
languages=[NN],
)
NORNE_NB_CONFIG = DatasetConfig(
name="norne-nb",
pretty_name="the truncated version of the Bokmål part of the Norwegian named "
"entity recognition dataset NorNE",
huggingface_id="EuroEval/norne-nb-mini",
task=NER,
languages=[NB, NO],
)
NORNE_NN_CONFIG = DatasetConfig(
name="norne-nn",
pretty_name="the truncated version of the Nynorsk part of the Norwegian named "
"entity recognition dataset NorNE",
huggingface_id="EuroEval/norne-nn-mini",
task=NER,
languages=[NN],
)
NORQUAD_CONFIG = DatasetConfig(
name="norquad",
pretty_name="the truncated version of the Norwegian question answering "
"dataset NorQuAD",
huggingface_id="EuroEval/norquad-mini",
task=RC,
languages=[NB, NN, NO],
_num_few_shot_examples=2,
)
NO_SAMMENDRAG_CONFIG = DatasetConfig(
name="no-sammendrag",
pretty_name="the truncated version of the Norwegian summarisation dataset "
"Norske Sammendrag",
huggingface_id="EuroEval/no-sammendrag-mini",
task=SUMM,
languages=[NB, NN, NO],
)
NRK_QUIZ_QA_CONFIG = DatasetConfig(
name="nrk-quiz-qa",
pretty_name="the truncated version of the Norwegian knowledge dataset NRK Quiz QA",
huggingface_id="EuroEval/nrk-quiz-qa-mini",
task=KNOW,
languages=[NB, NN, NO],
)
NOR_COMMON_SENSE_QA_CONFIG = DatasetConfig(
name="nor-common-sense-qa",
pretty_name="the truncated version of the Norwegian common-sense reasoning dataset "
"NorCommonSenseQA",
huggingface_id="EuroEval/nor-common-sense-qa",
task=COMMON_SENSE,
languages=[NB, NN, NO],
_labels=["a", "b", "c", "d", "e"],
)
### Unofficial datasets ###
NO_COLA_CONFIG = DatasetConfig(
name="no-cola",
pretty_name="the truncated version of the Norwegian linguistic acceptability "
"dataset NoCoLA",
huggingface_id="EuroEval/no-cola-mini",
task=LA,
languages=[NB, NO],
unofficial=True,
)
NORGLM_MULTI_QA = DatasetConfig(
name="norglm-multi-qa",
pretty_name="the question answering part of the Norwegian NorGLM multi-task human "
"annotated dataset NO-Multi-QA-Sum",
huggingface_id="EuroEval/norglm-multi-qa",
task=RC,
languages=[NB, NN, NO],
unofficial=True,
)
NORGLM_MULTI_SUM = DatasetConfig(
name="norglm-multi-sum",
pretty_name="the summarisation part of the Norwegian NorGLM multi-task human "
"annotated dataset NO-Multi-QA-Sum",
huggingface_id="EuroEval/norglm-multi-sum",
task=SUMM,
languages=[NB, NN, NO],
unofficial=True,
)
SCHIBSTED_NO_CONFIG = DatasetConfig(
name="schibsted-no",
pretty_name="the Norwegian summarisation dataset Schibsted-no",
huggingface_id="EuroEval/schibsted-article-summaries-no",
task=SUMM,
languages=[NB, NN, NO],
unofficial=True,
)
PERSONAL_SUM_CONFIG = DatasetConfig(
name="personal-sum",
pretty_name="the Norwegian summarisation dataset personal-sum",
huggingface_id="EuroEval/personal-sum",
task=SUMM,
languages=[NB, NN, NO],
unofficial=True,
)
MMLU_NO_CONFIG = DatasetConfig(
name="mmlu-no",
pretty_name="the truncated version of the Norwegian knowledge dataset MMLU-no, "
"translated from the English MMLU dataset",
huggingface_id="EuroEval/mmlu-no-mini",
task=KNOW,
languages=[NB, NN, NO],
unofficial=True,
)
ARC_NO_CONFIG = DatasetConfig(
name="arc-no",
pretty_name="the truncated version of the Norwegian knowledge dataset ARC-no, "
"translated from the English ARC dataset",
huggingface_id="EuroEval/arc-no-mini",
task=KNOW,
languages=[NB, NN, NO],
unofficial=True,
)
HELLASWAG_NO_CONFIG = DatasetConfig(
name="hellaswag-no",
pretty_name="the truncated version of the Norwegian common-sense reasoning dataset "
"HellaSwag-no, translated from the English HellaSwag dataset",
huggingface_id="EuroEval/hellaswag-no-mini",
task=COMMON_SENSE,
languages=[NB, NN, NO],
unofficial=True,
)
BELEBELE_NO_CONFIG = DatasetConfig(
name="belebele-no",
pretty_name="the Norwegian multiple choice reading comprehension dataset "
"BeleBele-no, translated from the English BeleBele dataset",
huggingface_id="EuroEval/belebele-no-mini",
task=MCRC,
languages=[NB, NN, NO],
unofficial=True,
)
|