{"total":51,"checklists":[{"id":"agent-loop-prevention","primitive":"code_execution","description":"Detect and break infinite agent loops using step counters, state hashing, and hard ceilings","registry_refs":["none (stdlib only)"],"tags":[],"solves":[],"auth_required":false,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/agent-loop-prevention"},{"id":"authenticated-rest-api-call","primitive":"tool_calling/api","description":"Make authenticated REST API calls using Bearer token and API key header patterns","registry_refs":["httpx","requests"],"tags":["http","rest-api","authentication","bearer-token","api-key","httpx","requests"],"solves":["token in URL instead of header","missing Authorization prefix","sync httpx instead of requests","no timeout set","credentials in logs"],"auth_required":true,"verified":true,"last_verified":"2026-04-14","eval_result":"pass","url":"https://checklist.day/api/checklists/authenticated-rest-api-call"},{"id":"document-summarization","primitive":"tool_calling/api","description":"Summarize long documents using map-reduce chunking with LLM","registry_refs":["langchain","openai","tiktoken"],"tags":["summarization","map-reduce","chunking","langchain","openai","tiktoken","long-document","llm"],"solves":["sending full document exceeds context window","no chunk overlap causes lost context","missing final reduce step","token counting wrong","summary of summary loses fidelity"],"auth_required":false,"verified":true,"last_verified":"2026-04-15","eval_result":"pass","url":"https://checklist.day/api/checklists/document-summarization"},{"id":"filesystem-read-write-organize","primitive":"code_execution","description":"Read, write, move, and organize files on the local filesystem","registry_refs":["none (stdlib only)"],"tags":["filesystem","pathlib","atomic-write","file-io","stdlib","move","rename"],"solves":["partial writes on crash","retry-safe mkdir","broken os.path string concat","silent data loss"],"auth_required":false,"verified":true,"last_verified":"2026-04-13","eval_result":"null","url":"https://checklist.day/api/checklists/filesystem-read-write-organize"},{"id":"flight-price-tracking","primitive":"tool_calling/api","description":"Track flight prices via API with rate limiting, retry, and scheduled polling","registry_refs":["requests","schedule","requests-ratelimiter"],"tags":["flight","travel","price-tracking","rate-limiting","retry","scheduling","api","polling"],"solves":["hammering API without rate limiting","no retry on 429","missing price change detection","unbounded polling loop","no result persistence"],"auth_required":false,"verified":true,"last_verified":"2026-04-15","eval_result":"pass","url":"https://checklist.day/api/checklists/flight-price-tracking"},{"id":"git-clone-branch-pr","primitive":"code_execution","description":"PR body text","registry_refs":["gitpython","pygithub"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/git-clone-branch-pr"},{"id":"json-validation-and-transform","primitive":"code_execution","description":"Validate and transform JSON data against a schema using pydantic","registry_refs":["pydantic"],"tags":["pydantic","json","validation","schema","transform","serialization","data-modeling"],"solves":["extra fields silently accepted","deprecated parse_obj usage","missing field-level error messages"],"auth_required":false,"verified":true,"last_verified":"2026-04-13","eval_result":"null","url":"https://checklist.day/api/checklists/json-validation-and-transform"},{"id":"local-nlp-preprocessing","primitive":"code_execution","description":"Preprocess text locally using spaCy — tokenization, NER, POS tagging, lemmatization","registry_refs":["spacy"],"tags":["nlp","spacy","tokenization","ner","pos-tagging","lemmatization","text-preprocessing","local"],"solves":["wrong model name causes OSError","model not downloaded before import","using nlp() on entire corpus at once causes OOM","mixing spaCy v2 and v3 APIs","disabled pipeline components slow down inference"],"auth_required":false,"verified":true,"last_verified":"2026-04-15","eval_result":"pass","url":"https://checklist.day/api/checklists/local-nlp-preprocessing"},{"id":"openai-chat-completion","primitive":"code_execution","description":"Model to use for completion","registry_refs":["openai"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/openai-chat-completion"},{"id":"openai-embeddings","primitive":"code_execution","description":"OpenAI API key starting with sk-","registry_refs":["openai"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/openai-embeddings"},{"id":"openai-function-calling","primitive":"code_execution","description":"Model to use (must support tool use)","registry_refs":["openai"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/openai-function-calling"},{"id":"openai-retry-on-429","primitive":"code_execution","description":"Model to use","registry_refs":["openai"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/openai-retry-on-429"},{"id":"openai-streaming-response","primitive":"code_execution","description":"Model to use for streaming completion","registry_refs":["openai"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/openai-streaming-response"},{"id":"openai-structured-output","primitive":"code_execution","description":"Model to use (gpt-4o or gpt-4o-mini recommended for structured output)","registry_refs":["openai"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/openai-structured-output"},{"id":"pinecone-ingest-and-query","primitive":"tool_calling/api","description":"Ingest documents into Pinecone serverless and query by similarity","registry_refs":["pinecone","sentence-transformers"],"tags":["pinecone","vector-store","embeddings","semantic-search","retrieval","rag","serverless"],"solves":["legacy pinecone.init() pattern removed in v3","wrong index creation for serverless vs pod","upsert batching missing","query without namespace","dimension mismatch on upsert"],"auth_required":true,"verified":true,"last_verified":"2026-04-14","eval_result":"null","url":"https://checklist.day/api/checklists/pinecone-ingest-and-query"},{"id":"postgres-bulk-insert","primitive":"code_execution","description":"Number of rows to insert in the benchmark","registry_refs":["psycopg2-binary"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/postgres-bulk-insert"},{"id":"postgres-connect-and-ping","primitive":"code_execution","description":"PostgreSQL connection string e.g. \"postgresql://user:pass@host:5432/dbname\"","registry_refs":["psycopg2-binary"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/postgres-connect-and-ping"},{"id":"postgres-connect-crud","primitive":"code_execution","description":"Table name to use for CRUD test (will be created and dropped)","registry_refs":["psycopg2-binary","sqlalchemy"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/postgres-connect-crud"},{"id":"postgres-connection-pool","primitive":"code_execution","description":"Number of connections in the pool","registry_refs":["psycopg2-binary","sqlalchemy"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/postgres-connection-pool"},{"id":"postgres-create-table-idempotent","primitive":"code_execution","description":"Set to \"true\" to drop the table after verification (useful for tests)","registry_refs":["psycopg2-binary"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/postgres-create-table-idempotent"},{"id":"postgres-insert-or-upsert","primitive":"code_execution","description":"Table to use for upsert test (created and dropped automatically)","registry_refs":["psycopg2-binary"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/postgres-insert-or-upsert"},{"id":"postgres-json-column-query","primitive":"code_execution","description":"PostgreSQL connection string e.g. \"postgresql://user:pass@host:5432/dbname\"","registry_refs":["psycopg2-binary"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/postgres-json-column-query"},{"id":"postgres-query-with-params","primitive":"code_execution","description":"PostgreSQL connection string e.g. \"postgresql://user:pass@host:5432/dbname\"","registry_refs":["psycopg2-binary"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/postgres-query-with-params"},{"id":"postgres-run-migration","primitive":"code_execution","description":"PostgreSQL connection string e.g. \"postgresql://user:pass@host:5432/dbname\"","registry_refs":["alembic","sqlalchemy"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/postgres-run-migration"},{"id":"rag-security","primitive":"tool_calling/api","description":"Secure a RAG pipeline against prompt injection, context poisoning, and retrieval manipulation","registry_refs":["chromadb","langchain","openai"],"tags":["rag","security","prompt-injection","retrieval","vector-store","llm-security"],"solves":["prompt injection via poisoned documents","context stuffing","retrieval manipulation","PII leakage in responses"],"auth_required":true,"verified":true,"last_verified":"2026-04-14","eval_result":"pass","url":"https://checklist.day/api/checklists/rag-security"},{"id":"rate-limit-handling-with-retry","primitive":"code_execution","description":"Handle API rate limits with exponential backoff and jitter using tenacity","registry_refs":["tenacity","requests"],"tags":[],"solves":[],"auth_required":false,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/rate-limit-handling-with-retry"},{"id":"redis-cache-aside-pattern","primitive":"code_execution","description":"Redis connection URL e.g. \"redis://localhost:6379\"","registry_refs":["redis"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/redis-cache-aside-pattern"},{"id":"redis-connect-and-ping","primitive":"code_execution","description":"Redis connection URL e.g. \"redis://localhost:6379\" or \"rediss://user:pass@host:6380\"","registry_refs":["redis"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/redis-connect-and-ping"},{"id":"redis-pubsub-publish-subscribe","primitive":"code_execution","description":"Redis connection URL e.g. \"redis://localhost:6379\"","registry_refs":["redis"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/redis-pubsub-publish-subscribe"},{"id":"redis-rate-limit-sliding-window","primitive":"code_execution","description":"Redis connection URL e.g. \"redis://localhost:6379\"","registry_refs":["redis"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/redis-rate-limit-sliding-window"},{"id":"redis-set-get-with-ttl","primitive":"code_execution","description":"Redis connection URL e.g. \"redis://localhost:6379\"","registry_refs":["redis"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/redis-set-get-with-ttl"},{"id":"s3-bucket-policy-and-cors","primitive":"code_execution","description":"AWS region","registry_refs":["boto3"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/s3-bucket-policy-and-cors"},{"id":"s3-connect-and-list-buckets","primitive":"code_execution","description":"AWS region","registry_refs":["boto3"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/s3-connect-and-list-buckets"},{"id":"s3-delete-object","primitive":"code_execution","description":"AWS region","registry_refs":["boto3"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/s3-delete-object"},{"id":"s3-download-file","primitive":"code_execution","description":"AWS region","registry_refs":["boto3"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/s3-download-file"},{"id":"s3-generate-presigned-url","primitive":"code_execution","description":"AWS region","registry_refs":["boto3"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/s3-generate-presigned-url"},{"id":"s3-upload-file","primitive":"code_execution","description":"AWS region","registry_refs":["boto3"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/s3-upload-file"},{"id":"semantic-similarity-search","primitive":"code_execution","description":"Embed documents and find most similar ones using cosine search","registry_refs":["sentence-transformers","faiss-cpu"],"tags":["embeddings","faiss","sentence-transformers","vector-search","cosine-similarity","nlp"],"solves":["wrong faiss index type for cosine","float64 vs float32 dtype errors","embedding shape mismatch"],"auth_required":false,"verified":true,"last_verified":"2026-04-13","eval_result":"null","url":"https://checklist.day/api/checklists/semantic-similarity-search"},{"id":"shell-command-execution","primitive":"code_execution","description":"Execute shell commands safely, capture output, and handle errors","registry_refs":["none (stdlib only)"],"tags":["subprocess","shell","cli","command-execution","timeout","stdlib","security"],"solves":["shell injection via shell=True","infinite hangs without timeout","swallowed stderr"],"auth_required":false,"verified":true,"last_verified":"2026-04-13","eval_result":"null","url":"https://checklist.day/api/checklists/shell-command-execution"},{"id":"sql-natural-language-query","primitive":"tool_calling/api","description":"Convert natural language to SQL, execute against SQLite, return structured results","registry_refs":["openai","sqlalchemy"],"tags":["sql","sqlite","natural-language","text-to-sql","nlp","structured-data","sqlalchemy"],"solves":["SQL injection via LLM-generated queries","no schema grounding causes hallucinated columns","missing result validation","destructive query execution"],"auth_required":false,"verified":true,"last_verified":"2026-04-14","eval_result":"pass","url":"https://checklist.day/api/checklists/sql-natural-language-query"},{"id":"ssl-tls-certificate-verification","primitive":"code_execution","description":"Verify SSL/TLS certificates, check expiry, validate chain, inspect DNS before deployment","registry_refs":["requests","httpx"],"tags":["ssl","tls","certificate","dns","deployment","security","https","verification"],"solves":["expired cert not caught before deploy","verify=False left in production","self-signed cert in prod","cert chain incomplete","DNS mismatch","no expiry buffer check"],"auth_required":false,"verified":true,"last_verified":"2026-04-14","eval_result":"pass","url":"https://checklist.day/api/checklists/ssl-tls-certificate-verification"},{"id":"stripe-confirm-and-capture-payment","primitive":"code_execution","description":"Stripe secret key (sk_test_... for test mode)","registry_refs":["stripe"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/stripe-confirm-and-capture-payment"},{"id":"stripe-create-customer","primitive":"code_execution","description":"Stripe secret key (sk_test_... for test mode)","registry_refs":["stripe"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/stripe-create-customer"},{"id":"stripe-create-payment-intent","primitive":"code_execution","description":"Stripe secret key (sk_test_... for test mode)","registry_refs":["stripe"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/stripe-create-payment-intent"},{"id":"stripe-create-subscription","primitive":"code_execution","description":"Stripe Price ID to subscribe to (price_...)","registry_refs":["stripe"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/stripe-create-subscription"},{"id":"stripe-issue-refund","primitive":"code_execution","description":"Stripe secret key (sk_test_... for test mode)","registry_refs":["stripe"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/stripe-issue-refund"},{"id":"stripe-list-and-query-objects","primitive":"code_execution","description":"Stripe secret key (sk_test_... for test mode)","registry_refs":["stripe"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/stripe-list-and-query-objects"},{"id":"stripe-webhook-verify-and-parse","primitive":"code_execution","description":"Webhook signing secret from Stripe dashboard (whsec_...)","registry_refs":["stripe"],"tags":[],"solves":[],"auth_required":true,"verified":false,"last_verified":"null","eval_result":"null","url":"https://checklist.day/api/checklists/stripe-webhook-verify-and-parse"},{"id":"structured-data-extraction-from-pdf","primitive":"tool_calling/api","description":"Extract structured JSON fields from PDF documents using pydantic validation","registry_refs":["pymupdf","pydantic","openai"],"tags":["pdf","extraction","pydantic","structured-output","pymupdf","openai","invoice","document-parsing"],"solves":["fitz import error (pymupdf naming)","no validation on extracted fields","hallucinated fields not caught","multi-page PDF loses page context","None fields not handled"],"auth_required":false,"verified":true,"last_verified":"2026-04-15","eval_result":"pass","url":"https://checklist.day/api/checklists/structured-data-extraction-from-pdf"},{"id":"vector-store-ingest-and-query","primitive":"code_execution","description":"Ingest documents into a vector store and query them by similarity","registry_refs":["chromadb"],"tags":["chromadb","vector-store","embeddings","semantic-search","retrieval","rag"],"solves":["deprecated Client(Settings()) pattern","non-idempotent collection creation on retry"],"auth_required":false,"verified":true,"last_verified":"2026-04-13","eval_result":"pass","url":"https://checklist.day/api/checklists/vector-store-ingest-and-query"},{"id":"web-scraping-structured-extraction","primitive":"code_execution","description":"Fetch a webpage and extract structured data fields from HTML","registry_refs":["beautifulsoup4","requests"],"tags":["beautifulsoup4","requests","scraping","html-parsing","structured-extraction","web"],"solves":["missing parser arg causing non-deterministic behavior","hanging requests without timeout"],"auth_required":false,"verified":true,"last_verified":"2026-04-13","eval_result":"null","url":"https://checklist.day/api/checklists/web-scraping-structured-extraction"}]}