{"library":"onnxruntime-gpu","title":"ONNX Runtime (GPU)","description":"ONNX Runtime is a high-performance inference engine for ONNX models. The `onnxruntime-gpu` package provides GPU acceleration (e.g., via CUDA, ROCm) for these models, building on the core ONNX Runtime. It's actively developed by Microsoft, with frequent releases often aligned with new ONNX operator sets and performance improvements, currently at version 1.24.4.","language":"python","status":"active","last_verified":"Wed May 13","install":{"commands":["pip install onnxruntime-gpu"],"cli":null},"imports":["import onnxruntime as ort\nsession = ort.InferenceSession(...)"],"auth":{"required":false,"env_vars":[]},"quickstart":{"code":"import onnxruntime as ort\nimport numpy as np\nimport onnx\nfrom onnx import helper, TensorProto\nimport os\n\n# 1. Create a dummy ONNX model for demonstration\n# Define the graph (input, output, and node)\nX = helper.make_tensor_value_info('X', TensorProto.FLOAT, [None, 3])\nY = helper.make_tensor_value_info('Y', TensorProto.FLOAT, [None, 3])\nnode = helper.make_node('Relu', ['X'], ['Y'])\ngraph = helper.make_graph([node], 'simple_relu', [X], [Y])\nmodel = helper.make_model(graph, producer_name='onnx-example')\n\n# Save it to a temporary file\nmodel_path = \"simple_relu.onnx\"\nonnx.save(model, model_path)\n\n# 2. Load the model with GPU provider\ntry:\n    # Prioritize CUDAExecutionProvider for NVIDIA GPUs\n    # Fallback to CPUExecutionProvider if CUDA is not available or fails\n    session = ort.InferenceSession(\n        model_path,\n        providers=[\"CUDAExecutionProvider\", \"CPUExecutionProvider\"]\n    )\n    print(\"ONNX Runtime session created with providers:\", session.get_providers())\n    \n    # Prepare dummy input data\n    input_data = np.random.rand(1, 3).astype(np.float32)\n    \n    # Run inference\n    output = session.run(None, {'X': input_data})\n    print(\"Inference successful. Output shape:\", output[0].shape)\n\nexcept Exception as e:\n    print(f\"\\nError creating ONNX Runtime session or running inference: {e}\")\n    print(\"Make sure you have a compatible CUDA environment (or other GPU runtime) \")\n    print(\"and the correct onnxruntime-gpu package installed. \\n\")\n    print(\"If CUDA is not available, try removing 'CUDAExecutionProvider' from the providers list.\")\n\nfinally:\n    # Clean up the dummy model file\n    if os.path.exists(model_path):\n        os.remove(model_path)\n","lang":"python","description":"This quickstart demonstrates how to create a simple ONNX model, save it, and then load it into an `InferenceSession` configured to prioritize GPU (CUDA) execution. It includes error handling for common GPU setup issues.","tag":null,"tag_description":null,"last_tested":"2026-04-25","results":[{"runtime":"python:3.10-alpine","exit_code":1},{"runtime":"python:3.10-slim","exit_code":-1},{"runtime":"python:3.11-alpine","exit_code":1},{"runtime":"python:3.11-slim","exit_code":-1},{"runtime":"python:3.12-alpine","exit_code":1},{"runtime":"python:3.12-slim","exit_code":-1},{"runtime":"python:3.13-alpine","exit_code":1},{"runtime":"python:3.13-slim","exit_code":-1},{"runtime":"python:3.9-alpine","exit_code":1},{"runtime":"python:3.9-slim","exit_code":-1}]},"compatibility":{"tag":null,"tag_description":null,"last_tested":"2026-05-13","installed_version":"1.19.2","pypi_latest":"1.26.0","is_stale":true,"summary":{"python_range":"3.10–3.9","success_rate":25,"avg_install_s":12,"avg_import_s":0.29,"wheel_type":"wheel"},"results":[{"runtime":"python:3.10-alpine","python_version":"3.10","os_libc":"alpine (musl)","variant":"onnxruntime-gpu","exit_code":1,"wheel_type":null,"failure_reason":"build_error","import_side_effects":null,"install_time_s":null,"import_time_s":null,"mem_mb":null,"disk_size":null},{"runtime":"python:3.10-alpine","python_version":"3.10","os_libc":"alpine (musl)","variant":"onnxruntime-gpu","exit_code":1,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":null,"mem_mb":null,"disk_size":null},{"runtime":"python:3.10-slim","python_version":"3.10","os_libc":"slim (glibc)","variant":"onnxruntime-gpu","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":15,"import_time_s":0.22,"mem_mb":7.1,"disk_size":"616M"},{"runtime":"python:3.10-slim","python_version":"3.10","os_libc":"slim (glibc)","variant":"onnxruntime-gpu","exit_code":1,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":null,"mem_mb":null,"disk_size":null},{"runtime":"python:3.11-alpine","python_version":"3.11","os_libc":"alpine (musl)","variant":"onnxruntime-gpu","exit_code":1,"wheel_type":null,"failure_reason":"build_error","import_side_effects":null,"install_time_s":null,"import_time_s":null,"mem_mb":null,"disk_size":null},{"runtime":"python:3.11-alpine","python_version":"3.11","os_libc":"alpine (musl)","variant":"onnxruntime-gpu","exit_code":1,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":null,"mem_mb":null,"disk_size":null},{"runtime":"python:3.11-slim","python_version":"3.11","os_libc":"slim (glibc)","variant":"onnxruntime-gpu","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":10,"import_time_s":0.32,"mem_mb":7.9,"disk_size":"534M"},{"runtime":"python:3.11-slim","python_version":"3.11","os_libc":"slim (glibc)","variant":"onnxruntime-gpu","exit_code":1,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":null,"mem_mb":null,"disk_size":null},{"runtime":"python:3.12-alpine","python_version":"3.12","os_libc":"alpine (musl)","variant":"onnxruntime-gpu","exit_code":1,"wheel_type":null,"failure_reason":"build_error","import_side_effects":null,"install_time_s":null,"import_time_s":null,"mem_mb":null,"disk_size":null},{"runtime":"python:3.12-alpine","python_version":"3.12","os_libc":"alpine (musl)","variant":"onnxruntime-gpu","exit_code":1,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":null,"mem_mb":null,"disk_size":null},{"runtime":"python:3.12-slim","python_version":"3.12","os_libc":"slim (glibc)","variant":"onnxruntime-gpu","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":9.3,"import_time_s":0.35,"mem_mb":7.5,"disk_size":"522M"},{"runtime":"python:3.12-slim","python_version":"3.12","os_libc":"slim (glibc)","variant":"onnxruntime-gpu","exit_code":1,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":null,"mem_mb":null,"disk_size":null},{"runtime":"python:3.13-alpine","python_version":"3.13","os_libc":"alpine (musl)","variant":"onnxruntime-gpu","exit_code":1,"wheel_type":null,"failure_reason":"build_error","import_side_effects":null,"install_time_s":null,"import_time_s":null,"mem_mb":null,"disk_size":null},{"runtime":"python:3.13-alpine","python_version":"3.13","os_libc":"alpine (musl)","variant":"onnxruntime-gpu","exit_code":1,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":null,"mem_mb":null,"disk_size":null},{"runtime":"python:3.13-slim","python_version":"3.13","os_libc":"slim (glibc)","variant":"onnxruntime-gpu","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":9.4,"import_time_s":0.34,"mem_mb":8.1,"disk_size":"522M"},{"runtime":"python:3.13-slim","python_version":"3.13","os_libc":"slim (glibc)","variant":"onnxruntime-gpu","exit_code":1,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":null,"mem_mb":null,"disk_size":null},{"runtime":"python:3.9-alpine","python_version":"3.9","os_libc":"alpine (musl)","variant":"onnxruntime-gpu","exit_code":1,"wheel_type":null,"failure_reason":"build_error","import_side_effects":null,"install_time_s":null,"import_time_s":null,"mem_mb":null,"disk_size":null},{"runtime":"python:3.9-alpine","python_version":"3.9","os_libc":"alpine (musl)","variant":"onnxruntime-gpu","exit_code":1,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":null,"mem_mb":null,"disk_size":null},{"runtime":"python:3.9-slim","python_version":"3.9","os_libc":"slim (glibc)","variant":"onnxruntime-gpu","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":16.1,"import_time_s":0.24,"mem_mb":6.7,"disk_size":"774M"},{"runtime":"python:3.9-slim","python_version":"3.9","os_libc":"slim (glibc)","variant":"onnxruntime-gpu","exit_code":1,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":null,"mem_mb":null,"disk_size":null}]}}