{"library":"megatron-energon","title":"Megatron-Energon","description":"NVIDIA Megatron-Energon is a multi-modal data loader library for large-scale deep learning, particularly for training large language models (LLMs) and vision-language models. It supports tar-based WebDataset, JSONL files, and polylithic datasets, with features like caching, AV decoding, and FUSE mount. Current version is 7.3.2, with active development and a release cadence of roughly monthly.","language":"python","status":"active","last_verified":"Fri May 01","install":{"commands":["pip install megatron-energon","pip install megatron-energon[all]"],"cli":{"name":"megatron-energon","version":""}},"imports":["from megatron.energon import get_train_dataset","from megatron.energon import WorkerConfig","from megatron.energon import Webdataset"],"auth":{"required":false,"env_vars":[]},"quickstart":{"code":"import os\nfrom megatron.energon import get_train_dataset, WorkerConfig\n\n# Create a simple dataset\nworker_config = WorkerConfig(\n    rank=0,\n    world_size=1,\n    num_workers=2,\n)\n\ndataset = get_train_dataset(\n    path=os.environ.get('DATASET_PATH', 'path/to/dataset'),\n    worker_config=worker_config,\n    batch_size=4,\n    shuffle_buffer_size=100,\n)\n\nfor batch in dataset:\n    # Each batch is a dict with keys like 'rgb', 'json'\n    print(batch.keys())\n    break","lang":"python","description":"Minimal example loading a training dataset with Megatron-Energon.","tag":null,"tag_description":null,"last_tested":null,"results":[]},"compatibility":null}