From 8768bdc96eed5b78e3b08601f4c0bfa2984c7aa9 Mon Sep 17 00:00:00 2001 From: Preet Shah <48907607+preetsshah@users.noreply.github.com> Date: Tue, 28 Nov 2023 13:46:21 +0530 Subject: [PATCH 1/2] Update ingestion_dependency.sh --- ingestion/ingestion_dependency.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ingestion/ingestion_dependency.sh b/ingestion/ingestion_dependency.sh index 5b2b78d0d296..b534c562d94c 100755 --- a/ingestion/ingestion_dependency.sh +++ b/ingestion/ingestion_dependency.sh @@ -18,14 +18,15 @@ DB_USER=${DB_USER:-airflow_user} DB_SCHEME=${DB_SCHEME:-mysql+pymysql} DB_PASSWORD=${DB_PASSWORD:-airflow_pass} -DB_CONN="${DB_SCHEME}://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:${DB_PORT}/${AIRFLOW_DB}" - AIRFLOW_ADMIN_USER=${AIRFLOW_ADMIN_USER:-admin} AIRFLOW_ADMIN_PASSWORD=${AIRFLOW_ADMIN_PASSWORD:-admin} -OPENMETADATA_SERVER=${OPENMETADATA_SERVER:-"http://openmetadata-server:8585"} +DB_USER_VAR=`echo "${DB_USER}" | python3 -c "import urllib.parse; encoded_user = urllib.parse.quote(input()); print(encoded_user)"` +DB_PASSWORD_VAR=`echo "${DB_PASSWORD}" | python3 -c "import urllib.parse; encoded_user = urllib.parse.quote(input()); print(encoded_user)"` + +DB_CONN=`echo -n "${DB_SCHEME}://${DB_USER_VAR}:${DB_PASSWORD_VAR}@${DB_HOST}:${DB_PORT}/${AIRFLOW_DB}"` -sed -i "s#\(sql_alchemy_conn = \).*#\1${DB_CONN}#" /opt/airflow/airflow.cfg +export AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=$DB_CONN airflow db init From c480cc959132d375863f099f23628c5e5ddc492b Mon Sep 17 00:00:00 2001 From: Preet Shah Date: Wed, 29 Nov 2023 13:49:49 +0530 Subject: [PATCH 2/2] Add parameter for DB connection properties --- docker/development/docker-compose.yml | 7 +++++++ ingestion/ingestion_dependency.sh | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml index 7be9052bacde..7ed2e041446f 100644 --- a/docker/development/docker-compose.yml +++ b/docker/development/docker-compose.yml @@ -358,6 +358,13 @@ services: DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+pymysql} DB_USER: ${AIRFLOW_DB_USER:-airflow_user} DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass} + + # extra connection-string properties for the database + # EXAMPLE + # require SSL (only for Postgres) + # properties: "?sslmode=require" + DB_PROPERTIES: "" + entrypoint: /bin/bash command: - "/opt/airflow/ingestion_dependency.sh" diff --git a/ingestion/ingestion_dependency.sh b/ingestion/ingestion_dependency.sh index b534c562d94c..2d0d864c1896 100755 --- a/ingestion/ingestion_dependency.sh +++ b/ingestion/ingestion_dependency.sh @@ -17,6 +17,7 @@ AIRFLOW_DB=${AIRFLOW_DB:-airflow_db} DB_USER=${DB_USER:-airflow_user} DB_SCHEME=${DB_SCHEME:-mysql+pymysql} DB_PASSWORD=${DB_PASSWORD:-airflow_pass} +DB_PROPERTIES=${DB_PROPERTIES} AIRFLOW_ADMIN_USER=${AIRFLOW_ADMIN_USER:-admin} AIRFLOW_ADMIN_PASSWORD=${AIRFLOW_ADMIN_PASSWORD:-admin} @@ -24,7 +25,7 @@ AIRFLOW_ADMIN_PASSWORD=${AIRFLOW_ADMIN_PASSWORD:-admin} DB_USER_VAR=`echo "${DB_USER}" | python3 -c "import urllib.parse; encoded_user = urllib.parse.quote(input()); print(encoded_user)"` DB_PASSWORD_VAR=`echo "${DB_PASSWORD}" | python3 -c "import urllib.parse; encoded_user = urllib.parse.quote(input()); print(encoded_user)"` -DB_CONN=`echo -n "${DB_SCHEME}://${DB_USER_VAR}:${DB_PASSWORD_VAR}@${DB_HOST}:${DB_PORT}/${AIRFLOW_DB}"` +DB_CONN=`echo -n "${DB_SCHEME}://${DB_USER_VAR}:${DB_PASSWORD_VAR}@${DB_HOST}:${DB_PORT}/${AIRFLOW_DB}${DB_PROPERTIES}"` export AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=$DB_CONN