"""Independent check of the reference answers of the TradingCalc LLM trading-math benchmark.

Recomputes, with plain Python (statistics, NormalDist, bisection for implied volatility) and without importing any
TradingCalc code, the reference value of every task of the seven task types that had wrong model answers in the 2026-10
run, straight from the question text, and compares it with the `expected` value stored in the raw results file. A
reference that disagrees would mean the benchmark itself is wrong, not the model.

Usage:  python -I -X utf8 audit-references.py llm-math-2026-10.json
Without an argument it reads public/benchmarks/llm-math-2026-10.json, relative to the repository root. The raw results
file is at https://tradingcalc.io/benchmarks/llm-math-2026-10.json. The regular expressions read the question wording of
the 2026-10 task generator; a new task type needs its own branch.
"""
import sys

import json, re, math, statistics as st
from statistics import NormalDist

N = NormalDist()
d = json.load(open(sys.argv[1] if len(sys.argv) > 1 else 'public/benchmarks/llm-math-2026-10.json', encoding='utf-8'))
tasks = {t['id']: t for t in d['tasks']}
NUM = r'(\d+(?:\.\d+)?)'


def series(q):
    m = re.search(r'in %: (.*?)\.\s*What', q, re.S)
    return [float(x) for x in m.group(1).split(',')]


def bs_call(S, K, T, s):
    d1 = (math.log(S / K) + 0.5 * s * s * T) / (s * math.sqrt(T))
    d2 = d1 - s * math.sqrt(T)
    return S * N.cdf(d1) - K * N.cdf(d2), N.cdf(d1)


def iv_delta(S, K, days, price, basis):
    T = days / basis
    lo, hi = 1e-4, 5.0
    for _ in range(200):
        mid = (lo + hi) / 2
        if bs_call(S, K, T, mid)[0] < price:
            lo = mid
        else:
            hi = mid
    return bs_call(S, K, T, mid)[1], mid


def inverse_pnl(side, n, e, x, f):
    gross = n * (1 / e - 1 / x) if side == 'long' else n * (1 / x - 1 / e)
    return gross - f * n / e - f * n / x


def ref(t):
    q, ty = t['question'], t['type']
    if ty == 'r_var_from_returns':
        x = series(q)
        c = float(re.search(r'(\d+)% parametric', q).group(1)) / 100
        return -st.mean(x) + N.inv_cdf(c) * st.stdev(x)
    if ty == 'r_sharpe_daily':
        x = series(q)
        return st.mean(x) / st.stdev(x) * math.sqrt(365)
    if ty == 'sharpe_annualized':
        x = [float(v) for v in re.search(r'\): (.*?)\. Using', q, re.S).group(1).split(',')]
        return st.mean(x) / st.stdev(x) * math.sqrt(12)
    if ty == 'r_iv_then_delta':
        K, days, price, S = [float(v) for v in re.search(
            r'strike ' + NUM + r' expiring in (\d+) days trades at ' + NUM + r' while the underlying is at ' + NUM, q).groups()]
        return iv_delta(S, K, days, price, 365)[0]
    if ty == 'r_inverse_pnl_plain':
        m = re.search(r'went (short|long) ([\d,]+) contracts.*? at ' + NUM + r' and closed at ' + NUM + r', paying the ' + NUM + '%', q)
        return inverse_pnl(m.group(1), float(m.group(2).replace(',', '')), float(m.group(3)), float(m.group(4)), float(m.group(5)) / 100)
    if ty == 'inverse_pnl_fees':
        m = re.search(r'(short|long) position of (\d+) USD.*?opened at ' + NUM + r' and closed at ' + NUM + r'.*?fee of ' + NUM + '%', q, re.S)
        return inverse_pnl(m.group(1), float(m.group(2)), float(m.group(3)), float(m.group(4)), float(m.group(5)) / 100)
    if ty == 'inverse_liquidation':
        m = re.search(r'(short|long) position.*?opened at ' + NUM + r' with (\d+)x', q, re.S)
        side, e, lev = m.group(1), float(m.group(2)), float(m.group(3))
        mm = float(re.search(r'which is ' + NUM + '% of', q).group(1)) / 100
        return e / (1 - 1 / lev + mm) if side == 'short' else e / (1 + 1 / lev - mm)


SUPPORTED = {'r_var_from_returns', 'r_sharpe_daily', 'sharpe_annualized', 'r_iv_then_delta', 'r_inverse_pnl_plain', 'inverse_pnl_fees', 'inverse_liquidation'}

# The task types that had at least one wrong answer, read from the file itself; every task of those types is rechecked.
wrong_rows = [r for r in d['rows'] if r['grade'] == 'wrong']
types = sorted({r['type'] for r in wrong_rows})
missing = [ty for ty in types if ty not in SUPPORTED]
if missing:
    print('NO INDEPENDENT RECOMPUTE for task type(s) with wrong answers:', ', '.join(missing))
    print('Add a branch to ref() for each before quoting a reference check for this run.')
    sys.exit(2)

bad = tot = 0
for t in tasks.values():
    if t['type'] not in types:
        continue
    tot += 1
    try:
        r = ref(t)
        rel = abs(r - t['expected']) / abs(t['expected'])
    except Exception as e:  # the question wording no longer matches what this script reads
        bad += 1
        print('CANNOT RECOMPUTE', t['id'], type(e).__name__, e)
        continue
    if rel > 1e-7:
        bad += 1
        print('MISMATCH', t['id'], 'mine', r, 'ours', t['expected'], 'rel', rel)
print(f'independent recompute: {tot - bad}/{tot} reference values agree to 1e-7 ({len(types)} types with wrong answers)')

iv_wrong = [r for r in wrong_rows if r['type'] == 'r_iv_then_delta']
if iv_wrong:
    print()
    print('r_iv_then_delta: delta under other year bases vs the model answers that were wrong')
    for r in iv_wrong:
        q = tasks[r['taskId']]['question']
        K, days, price, S = [float(v) for v in re.search(
            r'strike ' + NUM + r' expiring in (\d+) days trades at ' + NUM + r' while the underlying is at ' + NUM, q).groups()]
        print(r['taskId'], 'expected', round(tasks[r['taskId']]['expected'], 4), 'model', r['answer'],
              {b: round(iv_delta(S, K, days, price, b)[0], 4) for b in (365, 360, 252)})

sys.exit(1 if bad else 0)