Update ingestion_dependency.sh (#14139)

* Update ingestion_dependency.sh * Add parameter for DB connection properties --------- Co-authored-by: Akash Jain <[email protected]> (cherry picked from commit 20cf8f6)
open-metadata · Dec 4, 2023 · 1aaca32 · 1aaca32
1 parent f630322
commit 1aaca32
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 4 deletions.
diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml
@@ -362,6 +362,19 @@ services:
       DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+pymysql}
       DB_USER: ${AIRFLOW_DB_USER:-airflow_user}
       DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass}
+
+      # extra connection-string properties for the database
+      # EXAMPLE 
+      # require SSL (only for Postgres)
+      # properties: "?sslmode=require"
+      DB_PROPERTIES: ""
+
+      # To test the lineage backend
+      # AIRFLOW__LINEAGE__BACKEND: airflow_provider_openmetadata.lineage.backend.OpenMetadataLineageBackend
+      # AIRFLOW__LINEAGE__AIRFLOW_SERVICE_NAME: local_airflow
+      # AIRFLOW__LINEAGE__OPENMETADATA_API_ENDPOINT: http://openmetadata-server:8585/api
+      # AIRFLOW__LINEAGE__JWT_TOKEN: ...
+
     entrypoint: /bin/bash
     command:
       - "/opt/airflow/ingestion_dependency.sh"

diff --git a/ingestion/ingestion_dependency.sh b/ingestion/ingestion_dependency.sh
@@ -17,15 +17,17 @@ AIRFLOW_DB=${AIRFLOW_DB:-airflow_db}
 DB_USER=${DB_USER:-airflow_user}
 DB_SCHEME=${DB_SCHEME:-mysql+pymysql}
 DB_PASSWORD=${DB_PASSWORD:-airflow_pass}
-
-DB_CONN="${DB_SCHEME}://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:${DB_PORT}/${AIRFLOW_DB}"
+DB_PROPERTIES=${DB_PROPERTIES}
 
 AIRFLOW_ADMIN_USER=${AIRFLOW_ADMIN_USER:-admin}
 AIRFLOW_ADMIN_PASSWORD=${AIRFLOW_ADMIN_PASSWORD:-admin}
 
-OPENMETADATA_SERVER=${OPENMETADATA_SERVER:-"http://openmetadata-server:8585"}
+DB_USER_VAR=`echo "${DB_USER}" | python3 -c "import urllib.parse; encoded_user = urllib.parse.quote(input()); print(encoded_user)"`
+DB_PASSWORD_VAR=`echo "${DB_PASSWORD}" | python3 -c "import urllib.parse; encoded_user = urllib.parse.quote(input()); print(encoded_user)"`
+
+DB_CONN=`echo -n "${DB_SCHEME}://${DB_USER_VAR}:${DB_PASSWORD_VAR}@${DB_HOST}:${DB_PORT}/${AIRFLOW_DB}${DB_PROPERTIES}"`
 
-sed -i "s#\(sql_alchemy_conn = \).*#\1${DB_CONN}#" /opt/airflow/airflow.cfg
+export AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=$DB_CONN
 
 airflow db init