{
  "harness": "scripts/research/models/embed_words_bench.py",
  "git_sha": "98544960779e4d71af72d6d272c8edc655b176f6",
  "dataset_version": "SimLex-999 (Hill et al. 2015, 999 pairs); WordSim-353 (Finkelstein et al. 2002, 353 pairs); MTEB(eng, v2)",
  "built_at": "2026-10-13",
  "model": "tuatara-embed-words-1",
  "model_revision": "1e03750c49e65c38",
  "metrics": {
    "simlex999/all/spearman": 0.2776,
    "simlex999/covered/spearman": 0.2858,
    "simlex999/all/covered_pairs": 945,
    "wordsim353/all/spearman": 0.2434,
    "wordsim353/covered/spearman": 0.2445,
    "wordsim353/all/covered_pairs": 350,
    "mteb-eng/test/avg": 0.4565,
    "mteb-eng/test/Classification": 0.5946,
    "mteb-eng/test/Clustering": 0.3198,
    "mteb-eng/test/PairClassification": 0.6929,
    "mteb-eng/test/Reranking": 0.3836,
    "mteb-eng/test/Retrieval": 0.2402,
    "mteb-eng/test/STS": 0.6593,
    "mteb-eng/test/Summarization": 0.2184
  },
  "previous_release": {},
  "published": {},
  "note": "Measured with the served encoder code (src/services/model_provider/embed.py, sha256 5c872987e9e37477) on the dictionary model revision 1e03750c49e65c38; constants fixed, nothing tuned on a test split. SimLex-999 and WordSim-353 are encoded as one-word texts; a word the model lacks scores cosine 0 in the 'all' rows. MTEB English v2: all 41 tasks, per-task main score, unweighted mean."
}