"""
Similarity-threshold sweep for a semantic cache, on public labelled data.

Question it answers: "If prompt A is already cached and prompt B arrives,
does the cache serve A's answer to B, and should it have?"

Method (reproducible, CPU only, a few minutes):
  * Data:    GLUE QQP validation split (Quora Question Pairs, crowd-labelled
             "is duplicate"), from https://huggingface.co/datasets/nyu-mll/glue
             (file qqp/validation-00000-of-00001.parquet).
  * Sample:  N = 5,000 pairs drawn with a fixed seed (13).
  * Score:   cosine similarity of the two questions' embeddings.
  * Served:  similarity >= threshold (the cache would return A's answer to B).
  * Metrics per threshold:
      duplicates_served  = served duplicates / all labelled duplicates
                           (the useful hit rate a perfect workload could get)
      false_hit_share    = served non-duplicates / all served pairs
                           (how many cache answers belong to a different question)
  * Models:  two small open embedding models via fastembed (ONNX, CPU):
             BAAI/bge-small-en-v1.5 and sentence-transformers/all-MiniLM-L6-v2.
  * Caveat:  QQP labels are noisy; some "non-duplicates" are acceptable
             paraphrases. The script also dumps every labelled non-duplicate
             with similarity >= 0.95 (bge) so the reader can audit them.

Usage:
  python -m venv venv && venv/bin/pip install fastembed pandas pyarrow
  venv/bin/python threshold_sweep.py qqp_validation.parquet out.json
"""
import json
import sys

import numpy as np
import pandas as pd
from fastembed import TextEmbedding

N = 5000
SEED = 13
MODELS = ["BAAI/bge-small-en-v1.5", "sentence-transformers/all-MiniLM-L6-v2"]
THRESHOLDS = [round(float(x), 2) for x in np.arange(0.70, 0.991, 0.01)]


def embed(model, texts):
    m = TextEmbedding(model_name=model)
    v = np.array(list(m.embed(texts, batch_size=128)), dtype=np.float32)
    return v / np.linalg.norm(v, axis=1, keepdims=True)


def main(parquet, out):
    df = pd.read_parquet(parquet).sample(n=N, random_state=SEED).reset_index(drop=True)
    lab = df.label.to_numpy()
    res = {"n_pairs": N, "seed": SEED, "dataset": "GLUE QQP validation",
           "duplicates": int(lab.sum()), "models": {}, "audit": []}
    for model in MODELS:
        a, b = embed(model, df.question1.tolist()), embed(model, df.question2.tolist())
        sim = (a * b).sum(axis=1)
        rows = []
        for t in THRESHOLDS:
            served = sim >= t
            n_served = int(served.sum())
            good = int((served & (lab == 1)).sum())
            rows.append({
                "threshold": t,
                # Rough guide only: assumes API Management's score-threshold is a
                # cosine distance (1 - similarity). Microsoft documents only a
                # 0-1 range where lower is stricter, so do not copy this value.
                "apim_score_threshold": round(1 - t, 2),
                "served_share_of_pairs": n_served / N,
                "duplicates_served": good / int(lab.sum()),
                "false_hit_share": (n_served - good) / n_served if n_served else 0.0,
            })
        res["models"][model] = rows
        if model.startswith("BAAI"):
            idx = np.where((sim >= 0.95) & (lab == 0))[0]
            res["audit"] = [{"sim": round(float(sim[i]), 3),
                             "a": df.question1[i], "b": df.question2[i]}
                            for i in idx[np.argsort(-sim[idx])]]
        print(model, "done")
    with open(out, "w") as f:
        json.dump(res, f, indent=1, ensure_ascii=False)


if __name__ == "__main__":
    main(sys.argv[1], sys.argv[2])
