[v2] Refactior task specific scores

[Samoed]

mteb/mteb/evaluation/evaluators/utils.py

Lines 356 to 372 in 724f553

	TASK_TO_HF_DATASET = {
	"Core17InstructionRetrieval": ("jhu-clsp/core17-instructions-mteb", False),
	"Robust04InstructionRetrieval": ("jhu-clsp/robust04-instructions-mteb", False),
	"News21InstructionRetrieval": ("jhu-clsp/news21-instructions-mteb", False),
	"mFollowIR": ("jhu-clsp/mfollowir-parquet-mteb", True),
	"mFollowIRCrossLingual": (
	"jhu-clsp/mfollowir-cross-lingual-parquet-mteb",
	True,
	),
	}
	hf_path, is_multilingual = TASK_TO_HF_DATASET[task_name]
	if is_multilingual:
	# figure out which of the languages this is: ["zho", "rus", "fas"]
	# gather the changed_qrels for each, and store the keys as a check
	for lang in ["zho", "rus", "fas"]:
	config_name = f"qrel_diff-{lang}"
	changed_qrels = {

mteb/mteb/evaluation/evaluators/utils.py

Lines 587 to 610 in 724f553

	task_scores = {}
	if task_name in ["NevIR"]:
	paired_score = paired_accuracy(qrels, results, scores)
	task_scores["paired_accuracy"] = paired_score
	if task_name in ["InstructIR"]:
	robustness_at_10_score = robustness_at_10(qrels, results, scores)
	task_scores["robustness_at_10"] = robustness_at_10_score
	if task_name in [
	"mFollowIR",
	"mFollowIRCrossLingual",
	"Robust04InstructionRetrieval",
	"Core17InstructionRetrieval",
	"News21InstructionRetrieval",
	]:
	p_mrr_and_consolidated_scores = evaluate_p_mrr_change(
	results, qrels, task_name, k_values
	)
	task_scores.update(p_mrr_and_consolidated_scores)
	if task_name in ["MindSmallReranking"]:
	take_max_over_subqueries = max_over_subqueries(qrels, results, k_values)
	task_scores.update(take_max_over_subqueries)