requirements.txt

# Core libraries
numpy>=1.23.5
pandas>=1.5.2
scipy>=1.9.3
ipykernel>=6.8.0
jupyterlab>=3.2.9

# Dev tools
pytest>=7.1.0
mypy>=0.910 # static type checking
black>=22.10.0
sqlfluff>=1.3.2
typer[all]>=0.3.2 # includes rich
virtualenv>=20.13.1
virtualfish>=2.5.5
fastapi[all]>=0.87.0
uvicorn[standard]>=0.20.0
flake8>=6.0.0
# rich>=12.6.0 # rich is a dependency of typer
typing-extensions>=4.0.0
pydantic>=1.8.2

# Data engineering
apache-airflow>=2.4.3
psycopg2>=2.9.3
psycopg2-binary>=2.9.5
SQLAlchemy>=1.4.40
pyarrow>=7.0.0
pyspark>=3.3.1
apache-flink>=1.16.0
kafka-python>=2.0.2
great-expectations>=0.13.38
elasticsearch>=7.17.7
elasticsearch-dsl>=7.4.0

# ML libraries
torch>=1.10.0
torch-metrics>=0.6.0
transformers>=4.12.5
datasets>=1.15.0
sentence-transformers>=2.0.0
tensorboard>=2.7.0
spacy>=3.2.0
scikit-learn>=1.0.1
statsmodels>=0.13.0
ray>=1.9.0

# Visualisation
altair>=4.2.0
matplotlib>=3.6.2
plotly-dash==2.6.2
plotly-express>=0.4.1