#!/usr/bin/env python3
"""Check the Tuatara blend vs Jev comparison on BANKING77 yourself.

Downloads three public files, lines them up row by row, and prints each system's
accuracy and the exact McNemar test between them. Python 3.8+, standard library only.

    python3 score.py

Sources:
  official test labels  https://raw.githubusercontent.com/PolyAI-LDN/task-specific-datasets/master/banking_data/test.csv
  Jev's answers         https://raw.githubusercontent.com/simonmesmith/jev-banking77-experiment/main/results/predictions.csv
  blend's answers       https://cymetica.com/static/research/jev-banking77/blend_predictions.csv
"""
import csv
import io
import math
import urllib.request

OFFICIAL = "https://raw.githubusercontent.com/PolyAI-LDN/task-specific-datasets/master/banking_data/test.csv"
JEV = "https://raw.githubusercontent.com/simonmesmith/jev-banking77-experiment/main/results/predictions.csv"
BLEND = "https://cymetica.com/static/research/jev-banking77/blend_predictions.csv"


def fetch(url):
    req = urllib.request.Request(url, headers={"User-Agent": "banking77-score/1.0"})
    with urllib.request.urlopen(req, timeout=60) as r:
        return list(csv.DictReader(io.StringIO(r.read().decode("utf-8"))))


def mcnemar_exact(b, c):
    """Two-sided exact binomial test on the discordant pairs."""
    n, k = b + c, min(b, c)
    tail = sum(math.comb(n, i) for i in range(k + 1)) / 2 ** n
    return min(1.0, 2 * tail)


official, jev, blend = fetch(OFFICIAL), fetch(JEV), fetch(BLEND)
truth = [r["category"] for r in official]
assert len(truth) == len(jev) == len(blend) == 3080, "expected 3,080 test rows in each file"
for i, (t, j, b, o) in enumerate(zip(truth, jev, blend, official)):
    assert j["id"] == b["id"] == f"test-{i:05d}", f"row {i}: ids out of order"
    assert j["truth"] == t and b["truth"] == t, f"row {i}: label mismatch with the official test set"
    assert b["text"] == o["text"], f"row {i}: text mismatch with the official test set"

jev_ok = [j["prediction"] == t for j, t in zip(jev, truth)]
blend_ok = [b["prediction"] == t for b, t in zip(blend, truth)]
n = len(truth)
print(f"test messages            {n:,}")
print(f"Jev correct              {sum(jev_ok):,}  ({100 * sum(jev_ok) / n:.2f}%)")
print(f"Tuatara blend correct    {sum(blend_ok):,}  ({100 * sum(blend_ok) / n:.2f}%)")
only_blend = sum(b and not j for b, j in zip(blend_ok, jev_ok))
only_jev = sum(j and not b for b, j in zip(blend_ok, jev_ok))
print(f"blend right, Jev wrong   {only_blend}")
print(f"Jev right, blend wrong   {only_jev}")
print(f"exact McNemar p          {mcnemar_exact(only_blend, only_jev):.2f}")