{"id":"openai-embeddings","version":"1.0.0","primitive":"code_execution","description":"OpenAI API key starting with sk-","registry_refs":["openai"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","next_check":"2026-07-30","eval_result":"null","eval_env":"null","mast":[],"ref":"https://arxiv.org/abs/2503.13657","inputs":[{"name":"OPENAI_API_KEY","required":true,"description":"OpenAI API key starting with sk-"}],"executable":"# ============================================\n# checklist:     openai-embeddings\n# version:       1.0.0\n# primitive:     code_execution\n# description:   Generate embeddings with OpenAI, verify dimensions, and handle batching limits\n# registry_refs: openai\n# auth_required: true\n# verified:      false\n# last_verified: null\n# next_check:    2026-07-30\n# eval_result:   null\n# eval_env:      null\n#\n# inputs:\n#   - name: OPENAI_API_KEY\n#     required: true\n#     description: OpenAI API key starting with sk-\n#\n# OUTPUTS:\n#   model            — embedding model used\n#   dimensions       — number of dimensions in embedding vector\n#   batch_size       — number of texts embedded in one call\n#   similarity_ok    — true if cosine similarity between similar texts is > 0.8\n#\n# MAST FAILURE MODES ADDRESSED:\n# FM-2.6 Reasoning-Action Mismatch         — correct model name and dimensions documented\n# FM-3.2 No or Incomplete Verification     — embedding dimensions verified, similarity tested\n# FM-3.3 Incorrect Verification            — cosine similarity used, not euclidean distance\n#\n# ref: https://arxiv.org/abs/2503.13657\n# ============================================\n\nimport sys\nimport os\nimport subprocess\nimport time\nimport urllib.request\nimport json\nimport math\n\n# ─────────────────────────────────────────\n# PRE_EXECUTION\n# ─────────────────────────────────────────\n\nfor attempt in range(2):\n    try:\n        req = urllib.request.Request(\n            \"https://checklist.day/api/registry/openai\",\n            headers={\"User-Agent\": \"checklist-agent/1.0\"}\n        )\n        with urllib.request.urlopen(req, timeout=10) as resp:\n            registry = json.loads(resp.read())\n            break\n    except Exception as e:\n        if attempt == 1:\n            print(f\"ABORT: registry unreachable — {e}\")\n            sys.exit(1)\n        time.sleep(2)\n\nwarnings = registry.get(\"warnings\", [])\nif warnings:\n    print(\"[openai] WARNINGS:\")\n    for w in warnings if isinstance(warnings, list) else [warnings]:\n        print(f\"  ⚠ {w}\")\n\n# ─────────────────────────────────────────\n# EXECUTION\n# ─────────────────────────────────────────\n\nsubprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"openai>=1.0.0\"])\n\nfrom openai import OpenAI\n\nOPENAI_API_KEY = os.environ.get(\"OPENAI_API_KEY\")\nif not OPENAI_API_KEY:\n    print(\"ABORT: OPENAI_API_KEY env var not set\")\n    sys.exit(1)\n\nclient = OpenAI(api_key=OPENAI_API_KEY)\n\n# FOOTGUN: model name is \"text-embedding-3-small\", not \"text-embedding-ada-002\" (legacy)\n# FOOTGUN: text-embedding-3-small = 1536 dims, text-embedding-3-large = 3072 dims\nMODEL = \"text-embedding-3-small\"\nEXPECTED_DIMS = 1536\n\ntexts = [\n    \"The cat sat on the mat\",\n    \"A feline rested on a rug\",   # semantically similar\n    \"The stock market crashed\",    # semantically different\n]\n\n# FOOTGUN: batch limit is 2048 inputs per request — don't exceed\n# FOOTGUN: input must be list of strings, not a single string\nresponse = client.embeddings.create(\n    model=MODEL,\n    input=texts,\n)\n\nembeddings = [item.embedding for item in response.data]\ndimensions = len(embeddings[0])\nbatch_size = len(response.data)\n\nprint(f\"  model:      {response.model}\")\nprint(f\"  dimensions: {dimensions}\")\nprint(f\"  batch size: {batch_size}\")\n\ndef cosine_similarity(a: list, b: list) -> float:\n    dot   = sum(x * y for x, y in zip(a, b))\n    mag_a = math.sqrt(sum(x * x for x in a))\n    mag_b = math.sqrt(sum(x * x for x in b))\n    return dot / (mag_a * mag_b) if mag_a and mag_b else 0.0\n\nsim_similar   = cosine_similarity(embeddings[0], embeddings[1])\nsim_different = cosine_similarity(embeddings[0], embeddings[2])\nsimilarity_ok = sim_similar > sim_different\n\nprint(f\"  sim(cat/feline):  {sim_similar:.3f}\")\nprint(f\"  sim(cat/stocks):  {sim_different:.3f}\")\nprint(f\"  similarity_ok:    {similarity_ok}\")\n\n# ─────────────────────────────────────────\n# POST_EXECUTION\n# ─────────────────────────────────────────\n\nassert dimensions == EXPECTED_DIMS, f\"FAIL: expected {EXPECTED_DIMS} dims, got {dimensions}\"\nassert batch_size == len(texts), f\"FAIL: expected {len(texts)} embeddings, got {batch_size}\"\nassert similarity_ok, f\"FAIL: similar texts ({sim_similar:.3f}) should score higher than different ({sim_different:.3f})\"\n\nresult = {\n    \"model\":        response.model,\n    \"dimensions\":   dimensions,\n    \"batch_size\":   batch_size,\n    \"similarity_ok\": similarity_ok,\n}\nprint(json.dumps(result, indent=2))\nprint(\"PASS\")\n"}