{"id":"openai-structured-output","version":"1.0.0","primitive":"code_execution","description":"Model to use (gpt-4o or gpt-4o-mini recommended for structured output)","registry_refs":["openai"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","next_check":"2026-07-30","eval_result":"null","eval_env":"null","mast":[],"ref":"https://arxiv.org/abs/2503.13657","inputs":[{"name":"OPENAI_API_KEY","required":true,"description":"OpenAI API key starting with sk-"},{"name":"OPENAI_MODEL","default":"gpt-4o-mini","required":false,"description":"Model to use (gpt-4o or gpt-4o-mini recommended for structured output)"}],"executable":"# ============================================\n# checklist:     openai-structured-output\n# version:       1.0.0\n# primitive:     code_execution\n# description:   Get structured JSON output from OpenAI using response_format with json_schema mode and json_object fallback\n# registry_refs: openai\n# auth_required: true\n# verified:      false\n# last_verified: null\n# next_check:    2026-07-30\n# eval_result:   null\n# eval_env:      null\n#\n# inputs:\n#   - name: OPENAI_API_KEY\n#     required: true\n#     description: OpenAI API key starting with sk-\n#   - name: OPENAI_MODEL\n#     required: false\n#     default: \"gpt-4o-mini\"\n#     description: Model to use (gpt-4o or gpt-4o-mini recommended for structured output)\n#\n# OUTPUTS:\n#   json_schema_ok   — true if json_schema mode returned valid structured output\n#   json_object_ok   — true if json_object mode returned parseable JSON\n#   schema_validated — true if response matched expected schema\n#\n# MAST FAILURE MODES ADDRESSED:\n# FM-1.1 Disobey Task Specification        — response_format enforces structure, not prompt alone\n# FM-2.6 Reasoning-Action Mismatch         — json_schema vs json_object modes documented\n# FM-3.3 Incorrect Verification            — schema fields verified by name and type, not just parseable\n#\n# ref: https://arxiv.org/abs/2503.13657\n# ============================================\n\nimport sys\nimport os\nimport subprocess\nimport time\nimport urllib.request\nimport json\n\n# ─────────────────────────────────────────\n# PRE_EXECUTION\n# ─────────────────────────────────────────\n\nfor attempt in range(2):\n    try:\n        req = urllib.request.Request(\n            \"https://checklist.day/api/registry/openai\",\n            headers={\"User-Agent\": \"checklist-agent/1.0\"}\n        )\n        with urllib.request.urlopen(req, timeout=10) as resp:\n            registry = json.loads(resp.read())\n            break\n    except Exception as e:\n        if attempt == 1:\n            print(f\"ABORT: registry unreachable — {e}\")\n            sys.exit(1)\n        time.sleep(2)\n\nwarnings = registry.get(\"warnings\", [])\nif warnings:\n    print(\"[openai] WARNINGS:\")\n    for w in warnings if isinstance(warnings, list) else [warnings]:\n        print(f\"  ⚠ {w}\")\n\n# ─────────────────────────────────────────\n# EXECUTION\n# ─────────────────────────────────────────\n\nsubprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"openai>=1.0.0\"])\n\nfrom openai import OpenAI\n\nOPENAI_API_KEY = os.environ.get(\"OPENAI_API_KEY\")\nOPENAI_MODEL   = os.environ.get(\"OPENAI_MODEL\", \"gpt-4o-mini\")\n\nif not OPENAI_API_KEY:\n    print(\"ABORT: OPENAI_API_KEY env var not set\")\n    sys.exit(1)\n\nclient = OpenAI(api_key=OPENAI_API_KEY)\n\nPROMPT = \"Extract: name='Alice', age=30, is_active=true. Return as JSON.\"\n\n# Method 1: json_schema mode (strict, recommended for gpt-4o+)\n# FOOTGUN: json_schema requires \"strict\": true and all properties in \"required\"\n# FOOTGUN: does NOT work with older models like gpt-3.5-turbo\njson_schema_ok   = False\nschema_validated = False\n\ntry:\n    response = client.chat.completions.create(\n        model=OPENAI_MODEL,\n        messages=[\n            {\"role\": \"system\", \"content\": \"Extract structured data from the user message.\"},\n            {\"role\": \"user\",   \"content\": PROMPT},\n        ],\n        response_format={\n            \"type\": \"json_schema\",\n            \"json_schema\": {\n                \"name\":   \"person\",\n                \"strict\": True,\n                \"schema\": {\n                    \"type\": \"object\",\n                    \"properties\": {\n                        \"name\":      {\"type\": \"string\"},\n                        \"age\":       {\"type\": \"integer\"},\n                        \"is_active\": {\"type\": \"boolean\"},\n                    },\n                    \"required\":             [\"name\", \"age\", \"is_active\"],\n                    \"additionalProperties\": False,\n                },\n            },\n        },\n        max_tokens=64,\n    )\n    content = response.choices[0].message.content\n    parsed  = json.loads(content)\n    json_schema_ok   = True\n    schema_validated = (\n        parsed.get(\"name\") == \"Alice\" and\n        parsed.get(\"age\") == 30 and\n        parsed.get(\"is_active\") is True\n    )\n    print(f\"  json_schema mode: {parsed}\")\nexcept Exception as e:\n    print(f\"  json_schema mode failed: {e}\")\n\n# Method 2: json_object mode (looser, works with more models)\n# FOOTGUN: must include the word \"JSON\" in the prompt or system message — otherwise API error\n# FOOTGUN: json_object does NOT validate against a schema — model may return any JSON shape\njson_object_ok = False\n\ntry:\n    response2 = client.chat.completions.create(\n        model=OPENAI_MODEL,\n        messages=[\n            {\"role\": \"system\", \"content\": \"Return valid JSON only.\"},\n            {\"role\": \"user\",   \"content\": PROMPT},\n        ],\n        response_format={\"type\": \"json_object\"},\n        max_tokens=64,\n    )\n    content2       = response2.choices[0].message.content\n    parsed2        = json.loads(content2)\n    json_object_ok = isinstance(parsed2, dict) and len(parsed2) > 0\n    print(f\"  json_object mode: {parsed2}\")\nexcept Exception as e:\n    print(f\"  json_object mode failed: {e}\")\n\n# ─────────────────────────────────────────\n# POST_EXECUTION\n# ─────────────────────────────────────────\n\nassert json_object_ok, \"FAIL: json_object mode did not return parseable JSON dict\"\n\nresult = {\n    \"json_schema_ok\":   json_schema_ok,\n    \"json_object_ok\":   json_object_ok,\n    \"schema_validated\": schema_validated,\n}\nprint(json.dumps(result, indent=2))\nprint(\"PASS\")\n"}