Adds SpokeN-100-English

docs/create_tasks_table.py

@@ -79,7 +79,7 @@ def create_task_lang_table(tasks: list[mteb.AbsTask], sort_by_sum=False) -> str:
             if lang in PROGRAMMING_LANGS:
                 lang = "code"
             if table_dict.get(lang) is None:
-                table_dict[lang] = {k: 0 for k in sorted(get_args(TASK_TYPE))}
+                table_dict[lang] = dict.fromkeys(sorted(get_args(TASK_TYPE)), 0)
             table_dict[lang][task.metadata.type] += 1
 
     ## Wrangle for polars

mteb/abstasks/TaskMetadata.py

@@ -66,6 +66,7 @@
     "Speaker Count Identification",
     "Spoken Digit Classification",
     "Gender Clustering",
+    "Vocal Sound Classification",
     "Rendered semantic textual similarity",
     "Sentiment Analysis",
     "Intent Classification",

mteb/abstasks/stratification.py

@@ -216,7 +216,7 @@ def _prepare_stratification(self, y: np.ndarray) -> tuple:
             [self.percentage_per_fold[i] * self.n_samples for i in range(self.n_splits)]
         )
         rows = sp.lil_matrix(y).rows
-        rows_used = {i: False for i in range(self.n_samples)}
+        rows_used = dict.fromkeys(range(self.n_samples), False)
         all_combinations = []
         per_row_combinations = [[] for i in range(self.n_samples)]
         samples_with_combination = {}

mteb/evaluation/evaluators/RetrievalEvaluator.py

@@ -261,7 +261,7 @@ def search_cross_encoder(
                 logging.info(
                     f"previous_results is None. Using all the documents to rerank: {len(corpus)}"
                 )
-                q_results = {doc_id: 0.0 for doc_id in corpus.keys()}
+                q_results = dict.fromkeys(corpus.keys(), 0.0)
             else:
                 q_results = self.previous_results[qid]
             # take the top-k only

mteb/leaderboard/table.py

@@ -200,7 +200,7 @@ def scores_to_tables(
     joint_table_style = (
         joint_table.style.format(
             {
-                **{column: "{:.2f}" for column in score_columns},
+                **dict.fromkeys(score_columns, "{:.2f}"),
                 "Rank (Borda)": "{:.0f}",
                 "Zero-shot": format_zero_shot,
             },

mteb/task_aggregation.py

@@ -109,8 +109,7 @@ def borda_count(
     results = results.to_legacy_dict()
     n_candidates = sum(len(revs) for revs in results.values())
     candidate_scores = {
-        model: {revision: 0.0 for revision in revisions}
-        for model, revisions in results.items()
+        model: dict.fromkeys(revisions, 0.0) for model, revisions in results.items()
     }
 
     tasks = defaultdict(list)  # {task_name: [(model, revision, score), ...]}

mteb/tasks/Audio/AudioClassification/__init__.py

@@ -10,6 +10,7 @@
 from .eng.MridinghamStroke import *
 from .eng.MridinghamTonic import *
 from .eng.NSynth import *
+from .eng.SpokeN import *
 from .eng.SpokenQAforIC import *
 from .eng.VoxCelebSA import *
 from .eng.VoxLingua107Top10 import *

mteb/tasks/Audio/AudioClassification/eng/SpokeN.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from mteb.abstasks.Audio.AbsTaskAudioClassification import (
+    AbsTaskAudioClassification,
+)
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+
+class SpokeNEnglishClassification(AbsTaskAudioClassification):
+    metadata = TaskMetadata(
+        name="SpokeNEnglish",
+        description="Human Sound Classification Dataset.",
+        reference="https://zenodo.org/records/10810044",
+        dataset={
+            "path": "mteb/SpokeN-100-English",
+            "revision": "afbff14d927de14412d8124502313ea6d9d140e0",
+        },
+        type="AudioClassification",
+        category="a2t",
+        eval_splits=["train"],
+        eval_langs=["eng-Latn"],
+        main_score="accuracy",
+        date=("2024-01-01", "2024-01-01"),
+        domains=["Spoken"],
+        task_subtypes=["Vocal Sound Classification"],
+        license="cc-by-sa-4.0",
+        annotations_creators="LM-generated",
+        dialect=[],
+        modalities=["audio"],
+        sample_creation="found",
+        bibtex_citation="""@misc{groh2024spoken100crosslingualbenchmarkingdataset,
+            title={SpokeN-100: A Cross-Lingual Benchmarking Dataset for The Classification of Spoken Numbers in Different Languages}, 
+            author={René Groh and Nina Goes and Andreas M. Kist},
+            year={2024},
+            eprint={2403.09753},
+            archivePrefix={arXiv},
+            primaryClass={cs.SD},
+            url={https://arxiv.org/abs/2403.09753}, 
+        }""",
+        descriptive_stats={
+            "n_samples": {"train": 3200},
+        },
+    )
+
+    audio_column_name: str = "audio"
+    label_column_name: str = "label"
+    samples_per_label: int = 32

mteb/tasks/Image/Any2AnyRetrieval/multilingual/WITT2IRetrieval.py

@@ -24,9 +24,9 @@
 def _load_wit_data(
     path: str, langs: list, splits: str, cache_dir: str = None, revision: str = None
 ):
-    corpus = {lang: {split: None for split in splits} for lang in langs}
-    queries = {lang: {split: None for split in splits} for lang in langs}
-    relevant_docs = {lang: {split: None for split in splits} for lang in langs}
+    corpus = {lang: dict.fromkeys(splits) for lang in langs}
+    queries = {lang: dict.fromkeys(splits) for lang in langs}
+    relevant_docs = {lang: dict.fromkeys(splits) for lang in langs}
 
     split = "test"
 

mteb/tasks/Image/Any2AnyRetrieval/multilingual/XFlickr30kCoT2IRetrieval.py

@@ -21,9 +21,9 @@
 def _load_xflickrco_data(
     path: str, langs: list, splits: str, cache_dir: str = None, revision: str = None
 ):
-    corpus = {lang: {split: None for split in splits} for lang in langs}
-    queries = {lang: {split: None for split in splits} for lang in langs}
-    relevant_docs = {lang: {split: None for split in splits} for lang in langs}
+    corpus = {lang: dict.fromkeys(splits) for lang in langs}
+    queries = {lang: dict.fromkeys(splits) for lang in langs}
+    relevant_docs = {lang: dict.fromkeys(splits) for lang in langs}
 
     split = "test"
 

mteb/tasks/Image/Any2AnyRetrieval/multilingual/XM3600T2IRetrieval.py

@@ -49,9 +49,9 @@
 def _load_xm3600_data(
     path: str, langs: list, splits: str, cache_dir: str = None, revision: str = None
 ):
-    corpus = {lang: {split: None for split in splits} for lang in langs}
-    queries = {lang: {split: None for split in splits} for lang in langs}
-    relevant_docs = {lang: {split: None for split in splits} for lang in langs}
+    corpus = {lang: dict.fromkeys(splits) for lang in langs}
+    queries = {lang: dict.fromkeys(splits) for lang in langs}
+    relevant_docs = {lang: dict.fromkeys(splits) for lang in langs}
 
     split = "test"
 

mteb/tasks/Retrieval/dan/TwitterHjerneRetrieval.py

@@ -87,9 +87,7 @@ def dataset_transform(self) -> None:
                         answer_id = str(text2id[a])
                     answer_ids.append(answer_id)
 
-                self.relevant_docs[split][query_id] = {
-                    answer_id: 1 for answer_id in answer_ids
-                }
+                self.relevant_docs[split][query_id] = dict.fromkeys(answer_ids, 1)
 
 
 def answers_to_list(example: dict) -> dict:

mteb/tasks/Retrieval/deu/GerDaLIRRetrieval.py

@@ -72,7 +72,8 @@ def load_data(self, **kwargs):
         self.corpus = {self._EVAL_SPLIT: {row["_id"]: row for row in corpus_rows}}
         self.relevant_docs = {
             self._EVAL_SPLIT: {
-                row["_id"]: {v: 1 for v in row["text"].split(" ")} for row in qrels_rows
+                row["_id"]: dict.fromkeys(row["text"].split(" "), 1)
+                for row in qrels_rows
             }
         }
 

mteb/tasks/Retrieval/deu/GermanDPRRetrieval.py

@@ -82,7 +82,7 @@ def load_data(self, **kwargs):
                 existing_docs=all_docs,
             )
             corpus.update(neg_docs)
-            relevant_docs[q_id] = {k: 1 for k in pos_docs}
+            relevant_docs[q_id] = dict.fromkeys(pos_docs, 1)
         corpus = {
             key: doc.get("title", "") + " " + doc["text"] for key, doc in corpus.items()
         }

mteb/tasks/Retrieval/eng/BrightRetrieval.py

@@ -75,11 +75,9 @@ def load_bright_data(
         cache_dir: str = None,
         revision: str = None,
     ):
-        corpus = {domain: {split: None for split in eval_splits} for domain in DOMAINS}
-        queries = {domain: {split: None for split in eval_splits} for domain in DOMAINS}
-        relevant_docs = {
-            domain: {split: None for split in eval_splits} for domain in DOMAINS
-        }
+        corpus = {domain: dict.fromkeys(eval_splits) for domain in DOMAINS}
+        queries = {domain: dict.fromkeys(eval_splits) for domain in DOMAINS}
+        relevant_docs = {domain: dict.fromkeys(eval_splits) for domain in DOMAINS}
 
         for domain in domains:
             domain_corpus = datasets.load_dataset(

mteb/tasks/Retrieval/multilingual/CUREv1Retrieval.py

@@ -120,15 +120,9 @@ def load_data(self, **kwargs):
         cache_dir = kwargs.get("cache_dir", None)
 
         # Iterate over splits and languages
-        corpus = {
-            language: {split: None for split in eval_splits} for language in languages
-        }
-        queries = {
-            language: {split: None for split in eval_splits} for language in languages
-        }
-        relevant_docs = {
-            language: {split: None for split in eval_splits} for language in languages
-        }
+        corpus = {language: dict.fromkeys(eval_splits) for language in languages}
+        queries = {language: dict.fromkeys(eval_splits) for language in languages}
+        relevant_docs = {language: dict.fromkeys(eval_splits) for language in languages}
         for split in eval_splits:
             # Since this is a cross-lingual dataset, the corpus and the relevant documents do not depend on the language
             split_corpus = self._load_corpus(split=split, cache_dir=cache_dir)

mteb/tasks/Retrieval/multilingual/MIRACLRetrieval.py

@@ -34,9 +34,9 @@
 def _load_miracl_data(
     path: str, langs: list, splits: str, cache_dir: str = None, revision: str = None
 ):
-    corpus = {lang: {split: None for split in splits} for lang in langs}
-    queries = {lang: {split: None for split in splits} for lang in langs}
-    relevant_docs = {lang: {split: None for split in splits} for lang in langs}
+    corpus = {lang: dict.fromkeys(splits) for lang in langs}
+    queries = {lang: dict.fromkeys(splits) for lang in langs}
+    relevant_docs = {lang: dict.fromkeys(splits) for lang in langs}
 
     split = _EVAL_SPLIT
 
@@ -156,9 +156,9 @@ def load_data(self, **kwargs):
 def _load_miracl_data_hard_negatives(
     path: str, langs: list, splits: str, cache_dir: str = None, revision: str = None
 ):
-    corpus = {lang: {split: None for split in splits} for lang in langs}
-    queries = {lang: {split: None for split in splits} for lang in langs}
-    relevant_docs = {lang: {split: None for split in splits} for lang in langs}
+    corpus = {lang: dict.fromkeys(splits) for lang in langs}
+    queries = {lang: dict.fromkeys(splits) for lang in langs}
+    relevant_docs = {lang: dict.fromkeys(splits) for lang in langs}
 
     split = _EVAL_SPLIT
 

mteb/tasks/Retrieval/multilingual/MultiLongDocRetrieval.py

@@ -32,9 +32,9 @@ def load_mldr_data(
     cache_dir: str = None,
     revision: str = None,
 ):
-    corpus = {lang: {split: None for split in eval_splits} for lang in langs}
-    queries = {lang: {split: None for split in eval_splits} for lang in langs}
-    relevant_docs = {lang: {split: None for split in eval_splits} for lang in langs}
+    corpus = {lang: dict.fromkeys(eval_splits) for lang in langs}
+    queries = {lang: dict.fromkeys(eval_splits) for lang in langs}
+    relevant_docs = {lang: dict.fromkeys(eval_splits) for lang in langs}
 
     for lang in langs:
         lang_corpus = datasets.load_dataset(

mteb/tasks/Retrieval/multilingual/NeuCLIR2022Retrieval.py

@@ -24,9 +24,9 @@ def load_neuclir_data(
     cache_dir: str | None = None,
     revision: str | None = None,
 ):
-    corpus = {lang: {split: None for split in eval_splits} for lang in langs}
-    queries = {lang: {split: None for split in eval_splits} for lang in langs}
-    relevant_docs = {lang: {split: None for split in eval_splits} for lang in langs}
+    corpus = {lang: dict.fromkeys(eval_splits) for lang in langs}
+    queries = {lang: dict.fromkeys(eval_splits) for lang in langs}
+    relevant_docs = {lang: dict.fromkeys(eval_splits) for lang in langs}
 
     for lang in langs:
         lang_corpus = datasets.load_dataset(
@@ -110,9 +110,9 @@ def load_neuclir_data_hard_negatives(
     revision: str | None = None,
 ):
     split = "test"
-    corpus = {lang: {split: None for split in eval_splits} for lang in langs}
-    queries = {lang: {split: None for split in eval_splits} for lang in langs}
-    relevant_docs = {lang: {split: None for split in eval_splits} for lang in langs}
+    corpus = {lang: dict.fromkeys(eval_splits) for lang in langs}
+    queries = {lang: dict.fromkeys(eval_splits) for lang in langs}
+    relevant_docs = {lang: dict.fromkeys(eval_splits) for lang in langs}
 
     for lang in langs:
         corpus_identifier = f"corpus-{lang}"

mteb/tasks/Retrieval/multilingual/NeuCLIR2023Retrieval.py

@@ -24,9 +24,9 @@ def load_neuclir_data(
     cache_dir: str | None = None,
     revision: str | None = None,
 ):
-    corpus = {lang: {split: None for split in eval_splits} for lang in langs}
-    queries = {lang: {split: None for split in eval_splits} for lang in langs}
-    relevant_docs = {lang: {split: None for split in eval_splits} for lang in langs}
+    corpus = {lang: dict.fromkeys(eval_splits) for lang in langs}
+    queries = {lang: dict.fromkeys(eval_splits) for lang in langs}
+    relevant_docs = {lang: dict.fromkeys(eval_splits) for lang in langs}
 
     for lang in langs:
         lang_corpus = datasets.load_dataset(
@@ -111,9 +111,9 @@ def load_neuclir_data_hard_negatives(
     revision: str | None = None,
 ):
     split = "test"
-    corpus = {lang: {split: None for split in eval_splits} for lang in langs}
-    queries = {lang: {split: None for split in eval_splits} for lang in langs}
-    relevant_docs = {lang: {split: None for split in eval_splits} for lang in langs}
+    corpus = {lang: dict.fromkeys(eval_splits) for lang in langs}
+    queries = {lang: dict.fromkeys(eval_splits) for lang in langs}
+    relevant_docs = {lang: dict.fromkeys(eval_splits) for lang in langs}
 
     for lang in langs:
         corpus_identifier = f"corpus-{lang}"

mteb/tasks/Retrieval/multilingual/XMarketRetrieval.py

@@ -54,7 +54,7 @@ def _load_xmarket_data(
         corpus[lang][split] = {row["_id"]: row for row in corpus_rows}
         queries[lang][split] = {row["_id"]: row["text"] for row in query_rows}
         relevant_docs[lang][split] = {
-            row["_id"]: {v: 1 for v in row["text"].split(" ")} for row in qrels_rows
+            row["_id"]: dict.fromkeys(row["text"].split(" "), 1) for row in qrels_rows
         }
 
     corpus = datasets.DatasetDict(corpus)

mteb/tasks/Retrieval/spa/SpanishPassageRetrievalS2P.py

@@ -82,7 +82,8 @@ def load_data(self, **kwargs):
         self.corpus = {"test": {row["_id"]: row for row in corpus_rows}}
         self.relevant_docs = {
             "test": {
-                row["_id"]: {v: 1 for v in row["text"].split(" ")} for row in qrels_rows
+                row["_id"]: dict.fromkeys(row["text"].split(" "), 1)
+                for row in qrels_rows
             }
         }
 

mteb/tasks/Retrieval/spa/SpanishPassageRetrievalS2S.py

@@ -79,7 +79,8 @@ def load_data(self, **kwargs):
         self.corpus = {"test": {row["_id"]: row for row in corpus_rows}}
         self.relevant_docs = {
             "test": {
-                row["_id"]: {v: 1 for v in row["text"].split(" ")} for row in qrels_rows
+                row["_id"]: dict.fromkeys(row["text"].split(" "), 1)
+                for row in qrels_rows
             }
         }
 

tests/test_reproducible_workflow.py

@@ -55,7 +55,7 @@ def test_validate_task_to_prompt_name(task_name: str | mteb.AbsTask):
     else:
         task_names = [task_name]
 
-    model_prompts = {task_name: "prompt_name" for task_name in task_names}
+    model_prompts = dict.fromkeys(task_names, "prompt_name")
     model_prompts |= {task_name + "-query": "prompt_name" for task_name in task_names}
     model_prompts |= {task_name + "-passage": "prompt_name" for task_name in task_names}
     model_prompts |= {