"""Synthetic eval: two systems, a judge, bootstrap CI, Cohen's kappa.

Every label is drawn in this file. Nothing here was graded by a model or a person.
"""

import random

SEED = 7
N = 40
BOOT = 2000


def kappa(human, judge):
    n = len(human)
    agree = sum(h == j for h, j in zip(human, judge))
    po = agree / n
    pe = 0.0
    for label in (0, 1):
        pe += (human.count(label) / n) * (judge.count(label) / n)
    return po, pe, (po - pe) / (1 - pe)


def main() -> None:
    rng = random.Random(SEED)
    # truth is balanced. system A misses more often than B. the judge copies
    # the human with a fixed error rate, independent of which system is right.
    truth = [i % 2 for i in range(N)]
    human = truth[:]
    a_correct = []
    b_correct = []
    judge = []
    for i, gold in enumerate(truth):
        a_ok = rng.random() < 0.70
        b_ok = rng.random() < 0.85
        a_correct.append(1 if a_ok else 0)
        b_correct.append(1 if b_ok else 0)
        judge.append(human[i] if rng.random() < 0.80 else 1 - human[i])

    acc_a = sum(a_correct) / N
    acc_b = sum(b_correct) / N
    diffs = []
    for _ in range(BOOT):
        idx = [rng.randrange(N) for _ in range(N)]
        da = sum(a_correct[i] for i in idx) / N
        db = sum(b_correct[i] for i in idx) / N
        diffs.append(db - da)
    diffs.sort()
    lo = diffs[int(0.025 * BOOT)]
    hi = diffs[int(0.975 * BOOT) - 1]
    po, pe, k = kappa(human, judge)
    print(f"n={N} seed={SEED} bootstrap={BOOT}")
    print(f"acc_a={acc_a:.3f} acc_b={acc_b:.3f} diff_b_minus_a={acc_b - acc_a:.3f}")
    print(f"bootstrap_95ci_diff={lo:.3f},{hi:.3f}")
    print(f"judge_human_po={po:.3f} pe={pe:.3f} cohen_kappa={k:.3f}")
    print("synthetic=true")


if __name__ == "__main__":
    main()
