{"id":27750,"library":"flowetl","title":"FlowETL","description":"FlowETL is a collection of Apache Airflow operators and sensors designed for use with FlowKit, a platform for analysing CDR data. The library provides ETL operators specific to FlowDB and FlowMachine. Current version: 1.34.0. Release cadence: irregular, tied to FlowKit releases.","status":"active","version":"1.34.0","language":"python","source_language":"en","source_url":"https://github.com/Flowminder/FlowKit","tags":["airflow","etl","flowkit","cdr","data-engineering"],"install":[{"cmd":"pip install flowetl","lang":"bash","label":"Install from PyPI"}],"dependencies":[{"reason":"FlowETL is an Airflow plugin providing operators and sensors; requires Airflow >=2.x.","package":"apache-airflow","optional":false},{"reason":"Many operators interact with FlowDB (a PostgreSQL-based CDR database).","package":"flowdb","optional":true}],"imports":[{"note":"CSVToFlowDBOperator is a custom operator in flowetl, not in core Airflow.","wrong":"from airflow.operators import CSVToFlowDBOperator","symbol":"CSVToFlowDBOperator","correct":"from flowetl.operators import CSVToFlowDBOperator"},{"note":"FlowETLSensor is provided by flowetl package.","wrong":"from airflow.sensors import FlowETLSensor","symbol":"FlowETLSensor","correct":"from flowetl.sensors import FlowETLSensor"}],"quickstart":{"code":"from flowetl.operators import CSVToFlowDBOperator\nfrom flowetl.sensors import FlowETLSensor\n\n# Example DAG (requires Airflow environment)\nwith DAG('flowetl_example', start_date=datetime(2023,1,1), schedule_interval=None) as dag:\n    ingest = CSVToFlowDBOperator(\n        task_id='ingest_csv',\n        conn_id='flowdb_default',\n        csv_path='/data/sample.csv',\n        table_name='events.cdr'\n    )\n    wait = FlowETLSensor(\n        task_id='wait_for_data',\n        table_name='events.cdr',\n        poke_interval=60\n    )\n    ingest >> wait","lang":"python","description":"A minimal DAG using FlowETL's CSVToFlowDBOperator to load a CSV into FlowDB, then a FlowETLSensor to wait for the data to be available."},"warnings":[{"fix":"Update Airflow to 2.10.5 and replace deprecated operators with their new equivalents.","message":"FlowETL 1.33.0+ requires Airflow 2.10.5. 'airflow.providers.postgres.operators.postgres.PostgresOperator' is deprecated; use 'airflow.providers.postgres.operators.postgres.PostgresOperator' replaced by 'airflow.providers.common.sql.operators.sql.SQLExecuteQueryOperator'.","severity":"breaking","affected_versions":">=1.33.0"},{"fix":"Define the Airflow connection (e.g., via UI or environment variable) matching the conn_id used in the operator.","message":"FlowETL operators expect a specific connection id (e.g., 'flowdb_default') configured in Airflow. If not set, tasks fail with 'Connection not found'.","severity":"gotcha","affected_versions":"all"},{"fix":"Upgrade Python environment to 3.11 or later.","message":"FlowETL 1.30.0+ requires Python >=3.11. Older versions of Python are no longer supported.","severity":"deprecated","affected_versions":">=1.30.0"}],"env_vars":null,"last_verified":"2026-05-09T00:00:00.000Z","next_check":"2026-08-07T00:00:00.000Z","problems":[{"fix":"Run 'pip install flowetl' in the same Python environment as Airflow.","cause":"Package not installed or installed in wrong environment.","error":"ModuleNotFoundError: No module named 'flowetl'"},{"fix":"Create an Airflow connection with conn_id='flowdb_default' (type Postgres) via UI or CLI: `airflow connections add flowdb_default --conn-type postgres --conn-host ...`","cause":"Required Airflow connection is missing.","error":"airflow.exceptions.AirflowException: The conn_id `flowdb_default` isn't defined"}],"ecosystem":"pypi","meta_description":null,"install_score":null,"install_tag":null,"quickstart_score":null,"quickstart_tag":null}