{"id":21531,"library":"litdata","title":"LitData","description":"A high-performance data processing library for AI workflows, part of the Lightning AI ecosystem. Provides optimized streaming datasets and data loaders for training deep learning models. Current version: 0.2.61. Active development with frequent weekly releases.","status":"active","version":"0.2.61","language":"python","source_language":"en","source_url":"https://github.com/Lightning-AI/litdata","tags":["data-loading","deep-learning","streaming","lightning-ai","dataset"],"install":[{"cmd":"pip install litdata","lang":"bash","label":"pip"}],"dependencies":[{"reason":"Core dependency for data handling","package":"numpy","optional":false}],"imports":[{"note":"Old import path from earlier Lightning packages has been removed.","wrong":"from lightning.data import StreamingDataset","symbol":"StreamingDataset","correct":"from litdata import StreamingDataset"},{"note":"StreamingDataLoader is a top-level import.","wrong":"from litdata.streaming import StreamingDataLoader","symbol":"StreamingDataLoader","correct":"from litdata import StreamingDataLoader"},{"note":"optimize moved to top-level.","wrong":"from litdata.processing import optimize","symbol":"optimize","correct":"from litdata import optimize"},{"note":"","wrong":"","symbol":"LightningDataset","correct":"from litdata import LightningDataset"}],"quickstart":{"code":"from litdata import StreamingDataset, StreamingDataLoader\n\n# Create a simple streaming dataset\nclass MyDataset(StreamingDataset):\n    def __init__(self):\n        super().__init__(input_dir=\"s3://my-bucket/data\", shuffle=True)\n\ndataset = MyDataset()\ndataloader = StreamingDataLoader(dataset, batch_size=32)\nfor batch in dataloader:\n    print(batch)\n    break","lang":"python","description":"Example of using StreamingDataset with a dummy input directory. To use real data, replace the input_dir with a valid URI."},"warnings":[{"fix":"pip install litdata>=0.2.55","message":"In v0.2.55, writing compressed data to Lightning Storage directories was fixed. Previous versions could break. Upgrade to >=0.2.55 if using compressed output.","severity":"breaking","affected_versions":"<0.2.55"},{"fix":"Use StreamingDataset directly.","message":"The `LightningDataset` class may be deprecated in future versions in favor of `StreamingDataset`. Check release notes for migration.","severity":"deprecated","affected_versions":"all"},{"fix":"Use `optimize` from litdata to convert raw data into chunked format before streaming.","message":"StreamingDataset expects a specific directory structure. If you pass a path without properly chunked files, it may raise FileNotFoundError or hang. Always preprocess data using `optimize` function first.","severity":"gotcha","affected_versions":"all"}],"env_vars":null,"last_verified":"2026-04-27T00:00:00.000Z","next_check":"2026-07-26T00:00:00.000Z","problems":[{"fix":"Preprocess your data using `from litdata import optimize; optimize(...)` to create chunks. Ensure the input_dir points to a directory with .bin and .mtx files.","cause":"Input directory does not contain properly formatted chunk files or the path is incorrect.","error":"FileNotFoundError: No such file or directory"},{"fix":"Use `from litdata import StreamingDataset` instead of `from lightning.data import StreamingDataset`.","cause":"Attempting to import from the old package name 'lightning' instead of 'litdata'.","error":"ModuleNotFoundError: No module named 'lightning'"}],"ecosystem":"pypi","meta_description":null,"install_score":null,"install_tag":null,"quickstart_score":null,"quickstart_tag":null}