"""
Agent-readiness checks over public OpenAPI descriptions.

Reproducible method behind chart 1 of "Agent-Ready APIs" (conceptualise.de).

    /opt/anaconda3/bin/python -I openapi_sample.py <workdir>

1. Long tail: the APIs.guru OpenAPI Directory (api.apis.guru/v2/list.json).
   One entry per provider (random.Random(44)), 120 providers, preferred
   version, the directory's own JSON copy. Swagger 2.0 and OpenAPI 3.x.
2. Flagships: seven current vendor-published descriptions fetched on the run
   date (URLs below).

Every HTTP operation (get/put/post/patch/delete) is checked; per spec we
compute the share of operations that pass each check, then report the
median across specs (each API weighs the same) for the long tail, and the
per-spec share for each flagship. Only $refs to parameters, responses and
request bodies are resolved (one level); schemas are not walked.
Read-only, unauthenticated GETs only.
"""
import json
import os
import random
import re
import statistics
import sys
import urllib.request

FLAGSHIPS = {
    "Stripe": "https://raw.githubusercontent.com/stripe/openapi/master/openapi/spec3.json",
    "GitHub": "https://raw.githubusercontent.com/github/rest-api-description/main/descriptions/api.github.com/api.github.com.json",
    "Twilio": "https://raw.githubusercontent.com/twilio/twilio-oai/main/spec/json/twilio_api_v2010.json",
    "Box": "https://raw.githubusercontent.com/box/box-openapi/main/openapi.json",
    "PagerDuty": "https://raw.githubusercontent.com/PagerDuty/api-schema/main/reference/REST/openapiv3.json",
    "Jira Cloud": "https://developer.atlassian.com/cloud/jira/platform/swagger-v3.v3.json",
    "Adyen Checkout": "https://raw.githubusercontent.com/Adyen/adyen-openapi/main/json/CheckoutService-v72.json",
}
N_SAMPLE = 120
SEED = 44
METHODS = ("get", "put", "post", "patch", "delete")
PAGE_PARAMS = re.compile(
    r"^(\$?top|\$?skip|limit|offset|page|per_page|perpage|page_size|pagesize|"
    r"pagetoken|page_token|cursor|after|before|starting_after|ending_before|"
    r"maxresults|max_results|max|count|size|next|nexttoken|next_token|"
    r"continuation|continuationtoken|marker|start|startindex|start_at|startat|"
    r"maxitems|first|last|pagenumber|page\[.*\])$", re.I)
CHECKS = ["operation_id", "description", "params_described", "error_schema",
          "problem_json", "rate_limit_429", "idempotency_key", "pagination",
          "examples"]


def fetch(url, path):
    if not os.path.exists(path):
        req = urllib.request.Request(url, headers={"User-Agent": "openapi-sample/1.0"})
        with urllib.request.urlopen(req, timeout=120) as r, open(path, "wb") as f:
            f.write(r.read())
    with open(path, "rb") as f:
        raw = f.read()
    try:
        return json.loads(raw)
    except ValueError:
        import yaml
        return yaml.safe_load(raw)


def deref(spec, obj):
    if isinstance(obj, dict) and "$ref" in obj and isinstance(obj["$ref"], str) \
            and obj["$ref"].startswith("#/"):
        cur = spec
        for part in obj["$ref"][2:].split("/"):
            part = part.replace("~1", "/").replace("~0", "~")
            if not isinstance(cur, dict) or part not in cur:
                return {}
            cur = cur[part]
        return cur if isinstance(cur, dict) else {}
    return obj if isinstance(obj, dict) else {}


def has_example(o):
    return isinstance(o, dict) and ("example" in o or "examples" in o
                                    or "x-examples" in o)


def check_spec(spec):
    swagger2 = str(spec.get("swagger", "")).startswith("2")
    res = {c: [0, 0] for c in CHECKS}       # pass, applicable

    def mark(c, ok):
        res[c][1] += 1
        res[c][0] += 1 if ok else 0

    produces = spec.get("produces", []) or []
    for path, item in (spec.get("paths") or {}).items():
        if not isinstance(item, dict):
            continue
        item_params = [deref(spec, p) for p in item.get("parameters", []) or []]
        for m in METHODS:
            op = item.get(m)
            if not isinstance(op, dict):
                continue
            params = item_params + [deref(spec, p) for p in op.get("parameters", []) or []]
            mark("operation_id", bool(op.get("operationId")))
            mark("description", len((op.get("description") or "").strip()) >= 40)
            if params:
                mark("params_described", all((p.get("description") or "").strip()
                                             for p in params))
            resps = {str(k): deref(spec, v) for k, v in (op.get("responses") or {}).items()}
            err = {k: v for k, v in resps.items()
                   if k.startswith("4") or k.upper() == "4XX" or k == "default"}
            ok_schema, problem = False, False
            for v in err.values():
                if swagger2:
                    if v.get("schema"):
                        ok_schema = True
                    if any("problem+json" in p for p in (op.get("produces") or produces)):
                        problem = True
                else:
                    for mt, media in (v.get("content") or {}).items():
                        if isinstance(media, dict) and media.get("schema"):
                            ok_schema = True
                        if "problem+json" in mt:
                            problem = True
            mark("error_schema", ok_schema)
            mark("problem_json", problem)
            mark("rate_limit_429", "429" in resps)
            if m == "post":
                mark("idempotency_key", any(
                    p.get("in") == "header" and "idempot" in (p.get("name") or "").lower()
                    for p in params))
            if m == "get" and not path.rstrip("/").endswith("}"):
                mark("pagination", any(PAGE_PARAMS.match(p.get("name") or "")
                                       for p in params if p.get("in") == "query"))
            ex = has_example(op) or any(has_example(p) for p in params)
            rb = deref(spec, op.get("requestBody") or {})
            for media in (rb.get("content") or {}).values():
                ex = ex or has_example(media)
            for v in resps.values():
                ex = ex or has_example(v)
                for media in (v.get("content") or {}).values():
                    ex = ex or has_example(media)
            mark("examples", ex)
    ops = res["operation_id"][1]
    return ops, {c: (round(100.0 * a / b, 1) if b else None) for c, (a, b) in res.items()}


def main(work):
    os.makedirs(os.path.join(work, "guru"), exist_ok=True)
    lst = fetch("https://api.apis.guru/v2/list.json", os.path.join(work, "list.json"))
    byprov = {}
    for key in sorted(lst):
        byprov.setdefault(key.split(":")[0], []).append(key)
    rng = random.Random(SEED)
    provs = sorted(byprov)
    rng.shuffle(provs)
    sample, rows = [], []
    for prov in provs:
        if len(sample) >= N_SAMPLE:
            break
        key = rng.choice(byprov[prov])
        meta = lst[key]["versions"][lst[key]["preferred"]]
        fn = os.path.join(work, "guru", re.sub(r"[^A-Za-z0-9_.-]", "_", key) + ".json")
        try:
            spec = fetch(meta["swaggerUrl"], fn)
            ops, pct = check_spec(spec)
        except Exception as exc:  # noqa: BLE001
            print("skip", key, exc, file=sys.stderr)
            continue
        if ops == 0:
            continue
        sample.append(key)
        rows.append({"api": key, "updated": meta.get("updated", "")[:10],
                     "format": meta.get("openapiVer"), "operations": ops, **pct})
    tail = {}
    for c in CHECKS:
        vals = [r[c] for r in rows if r[c] is not None]
        tail[c] = {"median": round(statistics.median(vals), 1),
                   "mean": round(statistics.mean(vals), 1),
                   "specs_applicable": len(vals),
                   "specs_at_least_80": sum(v >= 80 for v in vals)}
    flag = {}
    for name, url in FLAGSHIPS.items():
        fn = os.path.join(work, "flag_" + re.sub(r"\W", "", name.lower()))
        ops, pct = check_spec(fetch(url, fn))
        flag[name] = {"url": url, "operations": ops, **pct}
    out = {"n_long_tail": len(rows), "long_tail": tail, "flagships": flag,
           "long_tail_specs": rows,
           "long_tail_updated_range": [min(r["updated"] for r in rows),
                                       max(r["updated"] for r in rows)],
           "long_tail_operations": sum(r["operations"] for r in rows)}
    with open(os.path.join(work, "results.json"), "w") as f:
        json.dump(out, f, indent=1)
    print(json.dumps({k: v for k, v in out.items() if k != "long_tail_specs"}, indent=1))


if __name__ == "__main__":
    main(sys.argv[1])
