105 lines
4.1 KiB
Python
105 lines
4.1 KiB
Python
#!/usr/bin/env python3
|
|
"""exp15 preflight — live /models listing fact-check (two-directions rule, $0).
|
|
|
|
Lists /models for every vendor whose key exists and flags the arm/judge/miner
|
|
slugs exp15 intends to use. LISTING ONLY — no generation, no billing. The alias
|
|
caveat still holds (00-provider-quirks: "not in /models" != "doesn't work"), so a
|
|
missing slug here is a flag to probe live, not a verdict.
|
|
|
|
Never prints keys. Persists raw listing to data for provenance (eval rule 1).
|
|
Run: eval/.venv/bin/python eval/exp15/preflight_models.py
|
|
"""
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import requests
|
|
from dotenv import load_dotenv
|
|
|
|
# eval/.env holds the keys; scripts read it themselves (never printed).
|
|
EVAL_DIR = Path(__file__).resolve().parent.parent
|
|
load_dotenv(EVAL_DIR / ".env")
|
|
|
|
OUT_DIR = Path("/home/ubuntu/books/gu-zhenren/exp15")
|
|
OUT_DIR.mkdir(parents=True, exist_ok=True)
|
|
RAW = OUT_DIR / "preflight_models.json"
|
|
|
|
# (name, base_url, env_key, [target substrings to flag])
|
|
PROVIDERS = [
|
|
("deepseek", "https://api.deepseek.com/v1", "DEEPSEEK_API_KEY",
|
|
["deepseek-v4-flash", "deepseek-v4-pro", "deepseek-chat", "deepseek-reasoner"]),
|
|
("zai", "https://api.z.ai/api/paas/v4", "ZAI_API_KEY",
|
|
["glm-5", "glm-5.1", "glm-5.2", "glm-4.7", "glm-4.6", "glm-4.5-air"]),
|
|
("xai", "https://api.x.ai/v1", "XAI_API_KEY",
|
|
["grok-4.3", "grok-4"]),
|
|
("gemini", "https://generativelanguage.googleapis.com/v1beta/openai", "GEMINI_API_KEY",
|
|
["gemini-3.1-pro", "gemini-3.5", "gemini-3.1-flash", "gemini-2.5"]),
|
|
("openai", "https://api.openai.com/v1", "OPENAI_API_KEY",
|
|
["gpt-5.4", "gpt-5-mini", "gpt-5"]),
|
|
("mistral", "https://api.mistral.ai/v1", "MISTRAL_API_KEY",
|
|
["mistral-large", "mistral-medium", "mistral-small"]),
|
|
("kimi", "https://api.moonshot.ai/v1", "KIMI_API_KEY",
|
|
["kimi-k2.6", "kimi-k2.5", "kimi-k2.7", "kimi-k2"]),
|
|
]
|
|
|
|
|
|
def list_models(name, base, env_key):
|
|
key = os.environ.get(env_key)
|
|
if not key:
|
|
return {"provider": name, "error": f"no key in env ({env_key})"}
|
|
url = base.rstrip("/") + "/models"
|
|
try:
|
|
r = requests.get(url, headers={"Authorization": f"Bearer {key}"}, timeout=40)
|
|
except Exception as e: # noqa: BLE001
|
|
return {"provider": name, "error": f"request failed: {type(e).__name__}: {e}"}
|
|
out = {"provider": name, "status": r.status_code, "url": url}
|
|
if r.status_code != 200:
|
|
out["body_snippet"] = r.text[:300]
|
|
return out
|
|
try:
|
|
data = r.json()
|
|
except Exception: # noqa: BLE001
|
|
out["error"] = "non-json body"
|
|
out["body_snippet"] = r.text[:300]
|
|
return out
|
|
items = data.get("data", data if isinstance(data, list) else [])
|
|
ids = sorted({(m.get("id") or m.get("name") or "") for m in items if isinstance(m, dict)})
|
|
out["count"] = len(ids)
|
|
out["model_ids"] = ids
|
|
return out
|
|
|
|
|
|
def main():
|
|
results = []
|
|
for name, base, env_key, targets in PROVIDERS:
|
|
res = list_models(name, base, env_key)
|
|
res["targets"] = targets
|
|
if "model_ids" in res:
|
|
ids = res["model_ids"]
|
|
res["target_hits"] = {t: [m for m in ids if t in m] for t in targets}
|
|
results.append(res)
|
|
time.sleep(0.3)
|
|
|
|
RAW.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
|
|
print(f"# exp15 preflight /models (raw -> {RAW})\n")
|
|
for res in results:
|
|
name = res["provider"]
|
|
if "error" in res:
|
|
print(f"[{name}] ERROR: {res['error']}")
|
|
continue
|
|
if res.get("status") != 200:
|
|
print(f"[{name}] HTTP {res['status']}: {res.get('body_snippet','')[:120]}")
|
|
continue
|
|
print(f"[{name}] HTTP 200, {res['count']} models")
|
|
for t, hits in res["target_hits"].items():
|
|
mark = "OK " if hits else "-- "
|
|
print(f" {mark}{t:<22} -> {hits if hits else 'NOT in /models (probe live before verdict)'}")
|
|
print("\nNOTE: alias caveat (00-provider-quirks calendar): a slug absent here may still serve (HTTP 200).")
|
|
print("Verdicts on slugs require BOTH this listing AND the vendor-doc workflow, per two-directions rule.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|